++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/logs/demand-backup-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/logs/demand-backup-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ grep '\-eks\-'
+++ jq -r .serverVersion.gitVersion
grep: warning: stray \ before -
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
++ '[' v1.31.13-gke.1023000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/sbin/sed -r 's/[^0-9.]+//g'
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.31
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ custom_port=27019
+ create_infra demand-backup-sharded-15678
+ local ns=demand-backup-sharded-15678
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.cnu20XFEZf
++ mktemp
+ local LAST_ERR=/tmp/tmp.YkmqhxXYDP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cnu20XFEZf
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.YkmqhxXYDP
+ rm /tmp/tmp.cnu20XFEZf /tmp/tmp.YkmqhxXYDP
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.5btj0F4Top
++ mktemp
+ local LAST_ERR=/tmp/tmp.KKcDSFYYmi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5btj0F4Top
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.KKcDSFYYmi
+ rm /tmp/tmp.5btj0F4Top /tmp/tmp.KKcDSFYYmi
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.JGBSO6TzDA
++ mktemp
+ local LAST_ERR=/tmp/tmp.QWDmhhH1yk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JGBSO6TzDA
+ cat /tmp/tmp.QWDmhhH1yk
+ rm /tmp/tmp.JGBSO6TzDA /tmp/tmp.QWDmhhH1yk
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nt1pBTZDGs
++ mktemp
+ local LAST_ERR=/tmp/tmp.2cMr6lkfQp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nt1pBTZDGs
+ cat /tmp/tmp.2cMr6lkfQp
+ rm /tmp/tmp.nt1pBTZDGs /tmp/tmp.2cMr6lkfQp
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.6XQxkeq4BK
++ mktemp
+ local LAST_ERR=/tmp/tmp.3X7VA5NYwa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6XQxkeq4BK
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.3X7VA5NYwa
+ rm /tmp/tmp.6XQxkeq4BK /tmp/tmp.3X7VA5NYwa
+ return 0
+ check_crd_for_deletion PR-2094-d1213e33
+ local git_tag=PR-2094-d1213e33
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2094-d1213e33/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hz2YPQD1kw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tFSxXwMfk9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.hz2YPQD1kw
++ cat /tmp/tmp.tFSxXwMfk9
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.hz2YPQD1kw
++ cat /tmp/tmp.tFSxXwMfk9
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.hz2YPQD1kw
++ cat /tmp/tmp.tFSxXwMfk9
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.hz2YPQD1kw
++ cat /tmp/tmp.tFSxXwMfk9
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.hz2YPQD1kw /tmp/tmp.tFSxXwMfk9
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
egrep: warning: egrep is obsolescent; using grep -E
+ local LAST_OUT=/tmp/tmp.TJBDp3x8UE
++ mktemp
+ local LAST_OUT=/tmp/tmp.n33rr29lDt
++ mktemp
+ local LAST_ERR=/tmp/tmp.pdISrAkei6
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.Psl7R71hMG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n33rr29lDt
+ cat /tmp/tmp.Psl7R71hMG
+ rm /tmp/tmp.n33rr29lDt /tmp/tmp.Psl7R71hMG
+ return 0
namespace "demand-backup-sharded-12513" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TJBDp3x8UE
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.pdISrAkei6
+ rm /tmp/tmp.TJBDp3x8UE /tmp/tmp.pdISrAkei6
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.oMx0J0HAbj
++ mktemp
+ local LAST_ERR=/tmp/tmp.WzJCRRNxhb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oMx0J0HAbj
+ cat /tmp/tmp.WzJCRRNxhb
+ rm /tmp/tmp.oMx0J0HAbj /tmp/tmp.WzJCRRNxhb
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jEIxQEuazt
++ mktemp
+ local LAST_ERR=/tmp/tmp.l43zg5yYGD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jEIxQEuazt
namespace/psmdb-operator created
+ cat /tmp/tmp.l43zg5yYGD
+ rm /tmp/tmp.jEIxQEuazt /tmp/tmp.l43zg5yYGD
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Qh9COr6fV1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J9pf8TMAJW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Qh9COr6fV1
++ cat /tmp/tmp.J9pf8TMAJW
++ rm /tmp/tmp.Qh9COr6fV1 /tmp/tmp.J9pf8TMAJW
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.cd88VhlQVu
++ mktemp
+ local LAST_ERR=/tmp/tmp.EsaRTQLAS2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cd88VhlQVu
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6" modified.
+ cat /tmp/tmp.EsaRTQLAS2
+ rm /tmp/tmp.cd88VhlQVu /tmp/tmp.EsaRTQLAS2
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2094-d1213e33'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2094-d1213e33
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.wqFFK3Xvz6
++ mktemp
+ local LAST_ERR=/tmp/tmp.r08Eq9wrYj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wqFFK3Xvz6
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.r08Eq9wrYj
+ rm /tmp/tmp.wqFFK3Xvz6 /tmp/tmp.r08Eq9wrYj
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hDF9bn4ebB
++ mktemp
+ local LAST_ERR=/tmp/tmp.sUMxygoRtm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hDF9bn4ebB
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.sUMxygoRtm
+ rm /tmp/tmp.hDF9bn4ebB /tmp/tmp.sUMxygoRtm
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2094-d1213e33") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.WS66nRIuNy
++ mktemp
+ local LAST_ERR=/tmp/tmp.I89aK1kV86
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WS66nRIuNy
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.I89aK1kV86
+ rm /tmp/tmp.WS66nRIuNy /tmp/tmp.I89aK1kV86
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DXaev5r22y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.w3nsinal7a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DXaev5r22y
++ cat /tmp/tmp.w3nsinal7a
++ rm /tmp/tmp.DXaev5r22y /tmp/tmp.w3nsinal7a
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-6b4f59fb84-hq7b2
+ local pod=percona-server-mongodb-operator-6b4f59fb84-hq7b2
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6b4f59fb84-hq7b2 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w728iQFXhK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nscluahIgG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w728iQFXhK
++ cat /tmp/tmp.nscluahIgG
++ rm /tmp/tmp.w728iQFXhK /tmp/tmp.nscluahIgG
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-6b4f59fb84-hq7b2
++ mktemp
+ local LAST_OUT=/tmp/tmp.GfsGBqG82w
++ mktemp
+ local LAST_ERR=/tmp/tmp.coAVkMdBq8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-6b4f59fb84-hq7b2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GfsGBqG82w
+ cat /tmp/tmp.coAVkMdBq8
+ rm /tmp/tmp.GfsGBqG82w /tmp/tmp.coAVkMdBq8
+ return 0
2025-10-17T20:21:35.021Z	INFO	setup	Manager starting up	{"gitCommit": "d1213e330eb33e2bd719f7d0e5ab9ed64bab87a4", "gitBranch": "PR-2094-d1213e33", "buildTime": "", "goVersion": "go1.25.3", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-sharded-15678
+ local namespace=demand-backup-sharded-15678
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-sharded-15678'
+ awk '{print$1}'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-sharded-15678
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-sharded-15678 --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.ykU9cLabT3
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_ERR=/tmp/tmp.eYe3DoI45B
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.7n3YOOGzcf
++ seq 0 2
+ for i in $(seq 0 2)
++ mktemp
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.BKbmQJPnGA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-sharded-15678 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ykU9cLabT3
+ cat /tmp/tmp.eYe3DoI45B
+ rm /tmp/tmp.ykU9cLabT3 /tmp/tmp.eYe3DoI45B
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7n3YOOGzcf
+ cat /tmp/tmp.BKbmQJPnGA
+ rm /tmp/tmp.7n3YOOGzcf /tmp/tmp.BKbmQJPnGA
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-sharded-15678
++ mktemp
+ local LAST_OUT=/tmp/tmp.Yd7X5J8e8R
++ mktemp
+ local LAST_ERR=/tmp/tmp.um8bozBuFW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-sharded-15678
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Yd7X5J8e8R
+ cat /tmp/tmp.um8bozBuFW
+ rm /tmp/tmp.Yd7X5J8e8R /tmp/tmp.um8bozBuFW
+ return 0
+ desc 'create namespace demand-backup-sharded-15678'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-sharded-15678
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-sharded-15678
++ mktemp
+ local LAST_OUT=/tmp/tmp.rqpnNVmHDt
++ mktemp
+ local LAST_ERR=/tmp/tmp.UTkNvMm6pR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-sharded-15678
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rqpnNVmHDt
namespace/demand-backup-sharded-15678 created
+ cat /tmp/tmp.UTkNvMm6pR
+ rm /tmp/tmp.rqpnNVmHDt /tmp/tmp.UTkNvMm6pR
+ return 0
+ set_kube_ctx demand-backup-sharded-15678
+ local namespace=demand-backup-sharded-15678
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kdUYJtTt8p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nsINZPRaJ5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kdUYJtTt8p
++ cat /tmp/tmp.nsINZPRaJ5
++ rm /tmp/tmp.kdUYJtTt8p /tmp/tmp.nsINZPRaJ5
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6 --namespace=demand-backup-sharded-15678
++ mktemp
+ local LAST_OUT=/tmp/tmp.QFdPKV7Rua
++ mktemp
+ local LAST_ERR=/tmp/tmp.IrI67ExkWM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6 --namespace=demand-backup-sharded-15678
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QFdPKV7Rua
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2094-d1213e33-2-cluster6" modified.
+ cat /tmp/tmp.IrI67ExkWM
+ rm /tmp/tmp.QFdPKV7Rua /tmp/tmp.IrI67ExkWM
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Fri Oct 17 20:22:13 2025
NAMESPACE: demand-backup-sharded-15678
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-sharded-15678.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-15678 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-15678

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-15678 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-15678 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Engm31Yqtz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ohUXh8ZMx8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Engm31Yqtz
++ cat /tmp/tmp.ohUXh8ZMx8
++ rm /tmp/tmp.Engm31Yqtz /tmp/tmp.ohUXh8ZMx8
++ return 0
+ MINIO_POD=minio-service-d9589b474-8pnq6
+ wait_pod minio-service-d9589b474-8pnq6
+ local pod=minio-service-d9589b474-8pnq6
+ set +o xtrace
waiting for pod/minio-service-d9589b474-8pnq6 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-15678.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.i6D1KCyynv
++ mktemp
+ local LAST_ERR=/tmp/tmp.QgNoUQTzD3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-15678.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i6D1KCyynv
service/minio-service created
+ cat /tmp/tmp.QgNoUQTzD3
+ rm /tmp/tmp.i6D1KCyynv /tmp/tmp.QgNoUQTzD3
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.iZ9FfOuPO3
++ mktemp
+ local LAST_ERR=/tmp/tmp.5imawlM24W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iZ9FfOuPO3
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-sharded-15678 namespace
+ cat /tmp/tmp.5imawlM24W
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-15678
+ rm /tmp/tmp.iZ9FfOuPO3 /tmp/tmp.5imawlM24W
+ return 0
+ desc 'create first PSMDB cluster with custom port 27019'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster with custom port 27019
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.QeWRDArGER
++ mktemp
+ local LAST_ERR=/tmp/tmp.MEWiSRNAov
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QeWRDArGER
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.MEWiSRNAov
+ rm /tmp/tmp.QeWRDArGER /tmp/tmp.MEWiSRNAov
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OtKW7bLM4T
++ mktemp
+ local LAST_ERR=/tmp/tmp.4e5wzPDwQw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OtKW7bLM4T
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.4e5wzPDwQw
+ rm /tmp/tmp.OtKW7bLM4T /tmp/tmp.4e5wzPDwQw
+ return 0
+ version_gt 1.19
++ echo '1.31 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/container-rc.yaml
+ /usr/sbin/sed s/docker/runc/g
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lgn1EVcGMY
++ mktemp
+ local LAST_ERR=/tmp/tmp.bzuoufVf5f
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lgn1EVcGMY
runtimeclass.node.k8s.io/container-rc unchanged
+ cat /tmp/tmp.bzuoufVf5f
+ rm /tmp/tmp.lgn1EVcGMY /tmp/tmp.bzuoufVf5f
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2094-d1213e33"'
+ local LAST_OUT=/tmp/tmp.BE1udLKWGI
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.WsXCVPpjOG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BE1udLKWGI
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.WsXCVPpjOG
+ rm /tmp/tmp.BE1udLKWGI /tmp/tmp.WsXCVPpjOG
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready......OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NG1tKwcbIl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FQzxliNBqT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NG1tKwcbIl
++ cat /tmp/tmp.FQzxliNBqT
++ rm /tmp/tmp.NG1tKwcbIl /tmp/tmp.FQzxliNBqT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ovlOSHiOP5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eVmjMY8gYl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ovlOSHiOP5
++ cat /tmp/tmp.eVmjMY8gYl
++ rm /tmp/tmp.ovlOSHiOP5 /tmp/tmp.eVmjMY8gYl
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1C9WM8j8wz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t3kX8fcqCg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1C9WM8j8wz
++ cat /tmp/tmp.t3kX8fcqCg
++ rm /tmp/tmp.1C9WM8j8wz /tmp/tmp.t3kX8fcqCg
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LaluPoG24p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MOJQyagdAm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LaluPoG24p
++ cat /tmp/tmp.MOJQyagdAm
++ rm /tmp/tmp.LaluPoG24p /tmp/tmp.MOJQyagdAm
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6VbunzGbgq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3rlwrEL844
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6VbunzGbgq
++ cat /tmp/tmp.3rlwrEL844
++ rm /tmp/tmp.6VbunzGbgq /tmp/tmp.3rlwrEL844
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YiTfE9ieG0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hNstFS2uWt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YiTfE9ieG0
++ cat /tmp/tmp.hNstFS2uWt
++ rm /tmp/tmp.YiTfE9ieG0 /tmp/tmp.hNstFS2uWt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kz3wR0DsG8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b3Xctz3UhO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kz3wR0DsG8
++ cat /tmp/tmp.b3Xctz3UhO
++ rm /tmp/tmp.kz3wR0DsG8 /tmp/tmp.b3Xctz3UhO
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.caCvYzN3de
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MpGjy93z83
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.caCvYzN3de
++ cat /tmp/tmp.MpGjy93z83
++ rm /tmp/tmp.caCvYzN3de /tmp/tmp.MpGjy93z83
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FacxFj6Lpd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GiTGQd3fnM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FacxFj6Lpd
++ cat /tmp/tmp.GiTGQd3fnM
++ rm /tmp/tmp.FacxFj6Lpd /tmp/tmp.GiTGQd3fnM
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.cY3TfFUgm9
++ mktemp
+ local LAST_ERR=/tmp/tmp.iZ0tpr31ln
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cY3TfFUgm9
+ cat /tmp/tmp.iZ0tpr31ln
+ rm /tmp/tmp.cY3TfFUgm9 /tmp/tmp.iZ0tpr31ln
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2025-10-17T20:26:49+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ compare_kubectl statefulset/some-name-rs1
+ local resource=statefulset/some-name-rs1
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-rs1.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs1
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ClljG5jQoz
++ mktemp
+ local LAST_ERR=/tmp/tmp.j8mBKkixip
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs1
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ClljG5jQoz
+ cat /tmp/tmp.j8mBKkixip
+ rm /tmp/tmp.ClljG5jQoz /tmp/tmp.j8mBKkixip
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs1.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs1.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs1.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs1.yml
+ log 'compare_kubectl: statefulset/some-name-rs1 OK'
+ set +o xtrace
[2025-10-17T20:26:49+0000] compare_kubectl: statefulset/some-name-rs1 OK
+ compare_kubectl statefulset/some-name-rs2
+ local resource=statefulset/some-name-rs2
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-rs2.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs2
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.iWFD4YKBUt
++ mktemp
+ local LAST_ERR=/tmp/tmp.4laX9mWPJd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs2
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iWFD4YKBUt
+ cat /tmp/tmp.4laX9mWPJd
+ rm /tmp/tmp.iWFD4YKBUt /tmp/tmp.4laX9mWPJd
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs2.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs2.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs2.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-rs2.yml
+ log 'compare_kubectl: statefulset/some-name-rs2 OK'
+ set +o xtrace
[2025-10-17T20:26:51+0000] compare_kubectl: statefulset/some-name-rs2 OK
+ compare_kubectl statefulset/some-name-cfg
+ local resource=statefulset/some-name-cfg
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-cfg.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-cfg
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RQqKKuPE3x
++ mktemp
+ local LAST_ERR=/tmp/tmp.7WzgfYUmrT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-cfg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RQqKKuPE3x
+ cat /tmp/tmp.7WzgfYUmrT
+ rm /tmp/tmp.RQqKKuPE3x /tmp/tmp.7WzgfYUmrT
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-cfg.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-cfg.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-cfg.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-cfg.yml
+ log 'compare_kubectl: statefulset/some-name-cfg OK'
+ set +o xtrace
[2025-10-17T20:26:51+0000] compare_kubectl: statefulset/some-name-cfg OK
+ compare_kubectl statefulset/some-name-mongos ''
+ local resource=statefulset/some-name-mongos
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EZxnmsjgHr
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZjfenvcfSn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EZxnmsjgHr
+ cat /tmp/tmp.ZjfenvcfSn
+ rm /tmp/tmp.EZxnmsjgHr /tmp/tmp.ZjfenvcfSn
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2025-10-17T20:26:53+0000] compare_kubectl: statefulset/some-name-mongos OK
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5vFGCaRNkr
++ mktemp
+ local LAST_ERR=/tmp/tmp.DqE3evL04l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5vFGCaRNkr
secret/some-name-mongos created
+ cat /tmp/tmp.DqE3evL04l
+ rm /tmp/tmp.5vFGCaRNkr /tmp/tmp.DqE3evL04l
+ return 0
+ sleep 10
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MoF48m31vM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mtRN2UCKTr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MoF48m31vM
++ cat /tmp/tmp.mtRN2UCKTr
++ rm /tmp/tmp.MoF48m31vM /tmp/tmp.mtRN2UCKTr
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RSmZBN4u3z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.11xIqQKqac
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RSmZBN4u3z
++ cat /tmp/tmp.11xIqQKqac
++ rm /tmp/tmp.RSmZBN4u3z /tmp/tmp.11xIqQKqac
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8gl1OZvvBa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4FJgrbAVNt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8gl1OZvvBa
++ cat /tmp/tmp.4FJgrbAVNt
++ rm /tmp/tmp.8gl1OZvvBa /tmp/tmp.4FJgrbAVNt
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........................................................................................
+ compare_kubectl statefulset/some-name-mongos -secret
+ local resource=statefulset/some-name-mongos
+ local postfix=-secret
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml
+ local new_result=/tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-mongos
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-sharded-15678", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.TejKhPRnK6
++ mktemp
+ local LAST_ERR=/tmp/tmp.bXU7gR1jqG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-mongos
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TejKhPRnK6
+ cat /tmp/tmp.bXU7gR1jqG
+ rm /tmp/tmp.TejKhPRnK6 /tmp/tmp.bXU7gR1jqG
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.2LHCki2JP7/statefulset_some-name-mongos.yml
+ log 'compare_kubectl: statefulset/some-name-mongos OK'
+ set +o xtrace
[2025-10-17T20:30:50+0000] compare_kubectl: statefulset/some-name-mongos OK
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U5FVo0wUeH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BpXliRoajw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U5FVo0wUeH
++ cat /tmp/tmp.BpXliRoajw
++ rm /tmp/tmp.U5FVo0wUeH /tmp/tmp.BpXliRoajw
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Pld3vT2aC
++ mktemp
+ local LAST_ERR=/tmp/tmp.GmVE96mkKD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9Pld3vT2aC
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("83a07864-c818-4dfc-a59b-0dbd24f4b271") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		},
		{
			"db" : "myApp1",
			"role" : "readWrite"
		},
		{
			"db" : "myApp2",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.GmVE96mkKD
+ rm /tmp/tmp.9Pld3vT2aC /tmp/tmp.GmVE96mkKD
+ return 0
+ run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp","rs0")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GBtcrQp8wL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.obPvYXbGuS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GBtcrQp8wL
++ cat /tmp/tmp.obPvYXbGuS
++ rm /tmp/tmp.GBtcrQp8wL /tmp/tmp.obPvYXbGuS
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.n3k3ebCwc5
++ mktemp
+ local LAST_ERR=/tmp/tmp.toLAokukqo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n3k3ebCwc5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4817b4ca-fa51-4260-8a42-37a21a446ded") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1760733056, 9),
		"signature" : {
			"hash" : BinData(0,"LWiZNhcNlRr/pJu19j96IxI6DGs="),
			"keyId" : NumberLong("7562289144454447126")
		}
	},
	"operationTime" : Timestamp(1760733056, 3)
}
bye
+ cat /tmp/tmp.toLAokukqo
+ rm /tmp/tmp.n3k3ebCwc5 /tmp/tmp.toLAokukqo
+ return 0
+ run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp1","rs1")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tYc3ibpbp7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bEJqgf30xv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tYc3ibpbp7
++ cat /tmp/tmp.bEJqgf30xv
++ rm /tmp/tmp.tYc3ibpbp7 /tmp/tmp.bEJqgf30xv
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4pWl5GmBEs
++ mktemp
+ local LAST_ERR=/tmp/tmp.dPwTF5eIg5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4pWl5GmBEs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("32ed6ce8-6725-440e-a0f2-2ad6bc58f713") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1760733059, 10),
		"signature" : {
			"hash" : BinData(0,"8XhkrOUJQn6SXfNeasWng+6nY5c="),
			"keyId" : NumberLong("7562289144454447126")
		}
	},
	"operationTime" : Timestamp(1760733059, 5)
}
bye
+ cat /tmp/tmp.dPwTF5eIg5
+ rm /tmp/tmp.4pWl5GmBEs /tmp/tmp.dPwTF5eIg5
+ return 0
+ run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=sh.enableSharding("myApp2","rs2")'
+ local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1253XquejJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y6yBMRz86K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1253XquejJ
++ cat /tmp/tmp.Y6yBMRz86K
++ rm /tmp/tmp.1253XquejJ /tmp/tmp.Y6yBMRz86K
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.icslfdAkQ1
++ mktemp
+ local LAST_ERR=/tmp/tmp.RolDy7FB3L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.icslfdAkQ1
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1a506f20-e981-4a27-8b7e-19d9566dd5ba") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
{
	"ok" : 1,
	"$clusterTime" : {
		"clusterTime" : Timestamp(1760733063, 9),
		"signature" : {
			"hash" : BinData(0,"SpcTLcGMU2M2Bl7UIveae0tZzUs="),
			"keyId" : NumberLong("7562289144454447126")
		}
	},
	"operationTime" : Timestamp(1760733063, 4)
}
bye
+ cat /tmp/tmp.RolDy7FB3L
+ rm /tmp/tmp.icslfdAkQ1 /tmp/tmp.RolDy7FB3L
+ return 0
+ insert_data_mongos 100500 myApp '' 27019
+ local data=100500
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i2dE3191Xg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zxh4WjA8uy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i2dE3191Xg
++ cat /tmp/tmp.zxh4WjA8uy
++ rm /tmp/tmp.i2dE3191Xg /tmp/tmp.zxh4WjA8uy
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MQX94arusk
++ mktemp
+ local LAST_ERR=/tmp/tmp.WsrGy6ICSD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MQX94arusk
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e6a33c81-e52f-484c-86e9-2947daa97615") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.WsrGy6ICSD
+ rm /tmp/tmp.MQX94arusk /tmp/tmp.WsrGy6ICSD
+ return 0
+ insert_data_mongos 100500 myApp1 '' 27019
+ local data=100500
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pb38ejxGKj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OLe2x11xVQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pb38ejxGKj
++ cat /tmp/tmp.OLe2x11xVQ
++ rm /tmp/tmp.pb38ejxGKj /tmp/tmp.OLe2x11xVQ
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6NttUHSfXs
++ mktemp
+ local LAST_ERR=/tmp/tmp.1rIl3dtRml
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6NttUHSfXs
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("fe63e8b8-eff6-4a10-b5e0-d04342783661") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1rIl3dtRml
+ rm /tmp/tmp.6NttUHSfXs /tmp/tmp.1rIl3dtRml
+ return 0
+ insert_data_mongos 100500 myApp2 '' 27019
+ local data=100500
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1VWBZCWtRE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zk9ScnQ58c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1VWBZCWtRE
++ cat /tmp/tmp.Zk9ScnQ58c
++ rm /tmp/tmp.1VWBZCWtRE /tmp/tmp.Zk9ScnQ58c
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YUqQHlCCpO
++ mktemp
+ local LAST_ERR=/tmp/tmp.CfWnYnaGMz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YUqQHlCCpO
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("36b83e3e-8fd0-42d8-bbfe-5b82533d07df") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.CfWnYnaGMz
+ rm /tmp/tmp.YUqQHlCCpO /tmp/tmp.CfWnYnaGMz
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' '' 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb '' '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lBKXZLmx96
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Dh0zV7InZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lBKXZLmx96
++ cat /tmp/tmp.9Dh0zV7InZ
++ rm /tmp/tmp.lBKXZLmx96 /tmp/tmp.9Dh0zV7InZ
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bxEk7SVjqv
++ mktemp
+ local LAST_ERR=/tmp/tmp.rFJGDSmfPk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bxEk7SVjqv
+ cat /tmp/tmp.rFJGDSmfPk
+ rm /tmp/tmp.bxEk7SVjqv /tmp/tmp.rFJGDSmfPk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2LHCki2JP7/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-10-17T20:26:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-10-17T20:26:10.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-10-17T20:26:07.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-0...2025-10-17T20:26:09.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-1...2025-10-17T20:26:11.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs1-2...2025-10-17T20:26:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-0...2025-10-17T20:26:12.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-1...2025-10-17T20:26:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs2-2...2025-10-17T20:26:09.000+0000 I listening for the commands

+ backup_name_aws=backup-aws-s3
+ backup_name_minio=backup-minio
+ backup_name_gcp=backup-gcp-cs
+ backup_name_azure=backup-azure-blob
+ desc 'run backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backups
-----------------------------------------------------------------------------------

+ run_backup minio
+ local storage=minio
+ local backup_name=backup-minio
+ local type=logical
+ desc 'run backup backup-minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-minio"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8kv39gDrRK
++ mktemp
+ local LAST_ERR=/tmp/tmp.B4anzTthju
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8kv39gDrRK
perconaservermongodbbackup.psmdb.percona.com/backup-minio created
+ cat /tmp/tmp.B4anzTthju
+ rm /tmp/tmp.8kv39gDrRK /tmp/tmp.B4anzTthju
+ return 0
+ '[' -z '' ']'
+ run_backup aws-s3
+ local storage=aws-s3
+ local backup_name=backup-aws-s3
+ local type=logical
+ desc 'run backup backup-aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-aws-s3
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-aws-s3"
			| .spec.storageName = "aws-s3"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XTU3OFHc4N
++ mktemp
+ local LAST_ERR=/tmp/tmp.5xGIRGcpBn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XTU3OFHc4N
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created
+ cat /tmp/tmp.5xGIRGcpBn
+ rm /tmp/tmp.XTU3OFHc4N /tmp/tmp.5xGIRGcpBn
+ return 0
+ run_backup gcp-cs
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs
+ local type=logical
+ desc 'run backup backup-gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-gcp-cs
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-gcp-cs"
			| .spec.storageName = "gcp-cs"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.H7Cs1E2dvh
++ mktemp
+ local LAST_ERR=/tmp/tmp.n1loKN4PzK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H7Cs1E2dvh
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created
+ cat /tmp/tmp.n1loKN4PzK
+ rm /tmp/tmp.H7Cs1E2dvh /tmp/tmp.n1loKN4PzK
+ return 0
+ run_backup azure-blob
+ local storage=azure-blob
+ local backup_name=backup-azure-blob
+ local type=logical
+ desc 'run backup backup-azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-azure-blob
-----------------------------------------------------------------------------------

+ yq eval '.metadata.name = "backup-azure-blob"
			| .spec.storageName = "azure-blob"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.yGGpNG2nGK
++ mktemp
+ local LAST_ERR=/tmp/tmp.bO55Rfi1Jj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yGGpNG2nGK
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created
+ cat /tmp/tmp.bO55Rfi1Jj
+ rm /tmp/tmp.yGGpNG2nGK /tmp/tmp.bO55Rfi1Jj
+ return 0
+ wait_backup backup-aws-s3
+ local backup_name=backup-aws-s3
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3 to reach ready state.........................................OK
+ wait_backup backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs to reach ready state................OK
+ wait_backup backup-azure-blob
+ local backup_name=backup-azure-blob
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob to reach ready state..OK
+ wait_backup backup-minio
+ local backup_name=backup-minio
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio to reach ready state..OK
+ sleep 5
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest backup-aws-s3
++ local backup_name=backup-aws-s3
++ kubectl_bin get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ sed 's|gs://||'
++ local LAST_OUT=/tmp/tmp.FXWQNrN2b0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TuXy0ickTC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-aws-s3 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FXWQNrN2b0
++ cat /tmp/tmp.TuXy0ickTC
++ rm /tmp/tmp.FXWQNrN2b0 /tmp/tmp.TuXy0ickTC
++ return 0
+ backup_dest_aws=operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z/rs0/myApp.test.gz
+ gunzip
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z/rs1/myApp1.test.gz
+ gunzip
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z/rs2/myApp2.test.gz
+ gunzip
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B8uL9w5Pfd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pPir2gaGcB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B8uL9w5Pfd
++ cat /tmp/tmp.pPir2gaGcB
++ rm /tmp/tmp.B8uL9w5Pfd /tmp/tmp.pPir2gaGcB
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pj8AteGz9o
++ mktemp
+ local LAST_ERR=/tmp/tmp.1Jr5tKgNOe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pj8AteGz9o
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("92b88da9-3cd3-4afa-ae98-b573b479658a") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.1Jr5tKgNOe
+ rm /tmp/tmp.Pj8AteGz9o /tmp/tmp.1Jr5tKgNOe
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NxKHcjfMc2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pzFaRFXeZv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NxKHcjfMc2
++ cat /tmp/tmp.pzFaRFXeZv
++ rm /tmp/tmp.NxKHcjfMc2 /tmp/tmp.pzFaRFXeZv
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FjcnkvcjhJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.NtYiE8DgSQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FjcnkvcjhJ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("3ea44c98-4f20-4c8d-b999-b7ef7cac1f25") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NtYiE8DgSQ
+ rm /tmp/tmp.FjcnkvcjhJ /tmp/tmp.NtYiE8DgSQ
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JA7TK22hMq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.geowz3ryCF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JA7TK22hMq
++ cat /tmp/tmp.geowz3ryCF
++ rm /tmp/tmp.JA7TK22hMq /tmp/tmp.geowz3ryCF
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4Wva2gGWbn
++ mktemp
+ local LAST_ERR=/tmp/tmp.49cxORLiAK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4Wva2gGWbn
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("0def6104-0cb1-45e7-89dd-b4b5c9f5292a") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.49cxORLiAK
+ rm /tmp/tmp.4Wva2gGWbn /tmp/tmp.49cxORLiAK
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
egrep: warning: egrep is obsolescent; using grep -E
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IGjJE1gX62
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DvFqQFn1n3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IGjJE1gX62
++ cat /tmp/tmp.DvFqQFn1n3
++ rm /tmp/tmp.IGjJE1gX62 /tmp/tmp.DvFqQFn1n3
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jJkeLcZqo6
++ mktemp
+ local LAST_ERR=/tmp/tmp.0pW8AMBQsr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jJkeLcZqo6
+ cat /tmp/tmp.0pW8AMBQsr
+ rm /tmp/tmp.jJkeLcZqo6 /tmp/tmp.0pW8AMBQsr
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.2LHCki2JP7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WbdHHOc1Sd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OtFmVAmIag
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WbdHHOc1Sd
++ cat /tmp/tmp.OtFmVAmIag
++ rm /tmp/tmp.WbdHHOc1Sd /tmp/tmp.OtFmVAmIag
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YuU4veoJdS
++ mktemp
+ local LAST_ERR=/tmp/tmp.CehO10GymA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YuU4veoJdS
+ cat /tmp/tmp.CehO10GymA
+ rm /tmp/tmp.YuU4veoJdS /tmp/tmp.CehO10GymA
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.2LHCki2JP7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kbdZmuMj3D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tMierITWUY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kbdZmuMj3D
++ cat /tmp/tmp.tMierITWUY
++ rm /tmp/tmp.kbdZmuMj3D /tmp/tmp.tMierITWUY
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jEocM5s2h6
++ mktemp
+ local LAST_ERR=/tmp/tmp.D2LjPbMxoC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jEocM5s2h6
+ cat /tmp/tmp.D2LjPbMxoC
+ rm /tmp/tmp.jEocM5s2h6 /tmp/tmp.D2LjPbMxoC
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.2LHCki2JP7/find-2nd2
+ run_restore backup-aws-s3
+ local backup_name=backup-aws-s3
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-aws-s3/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-aws-s3/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ypw9XgmLnA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bv7d28Z0GN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ypw9XgmLnA
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3 created
+ cat /tmp/tmp.Bv7d28Z0GN
+ rm /tmp/tmp.ypw9XgmLnA /tmp/tmp.Bv7d28Z0GN
+ return 0
+ wait_restore backup-aws-s3 some-name
+ local backup_name=backup-aws-s3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3 object to be created.OK
Waiting psmdb-restore/restore-backup-aws-s3 to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dEjLtuvFln
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yjyQYId2hM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dEjLtuvFln
++ cat /tmp/tmp.yjyQYId2hM
++ rm /tmp/tmp.dEjLtuvFln /tmp/tmp.yjyQYId2hM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cMFUnaAlv5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xpe5DrnneH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cMFUnaAlv5
++ cat /tmp/tmp.Xpe5DrnneH
++ rm /tmp/tmp.cMFUnaAlv5 /tmp/tmp.Xpe5DrnneH
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YI1yYiue9y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yiSoaA1b0X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YI1yYiue9y
++ cat /tmp/tmp.yiSoaA1b0X
++ rm /tmp/tmp.YI1yYiue9y /tmp/tmp.yiSoaA1b0X
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y0qmc5GDFm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C6ZqY8UEnt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y0qmc5GDFm
++ cat /tmp/tmp.C6ZqY8UEnt
++ rm /tmp/tmp.y0qmc5GDFm /tmp/tmp.C6ZqY8UEnt
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TnKp9UvO60
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zqSQj62RCV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TnKp9UvO60
++ cat /tmp/tmp.zqSQj62RCV
++ rm /tmp/tmp.TnKp9UvO60 /tmp/tmp.zqSQj62RCV
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.q8PVvZML77
++ mktemp
+ local LAST_ERR=/tmp/tmp.6GftD62eYc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q8PVvZML77
+ cat /tmp/tmp.6GftD62eYc
+ rm /tmp/tmp.q8PVvZML77 /tmp/tmp.6GftD62eYc
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2LHCki2JP7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CWFOFNyA1N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.atw4LTrZWw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CWFOFNyA1N
++ cat /tmp/tmp.atw4LTrZWw
++ rm /tmp/tmp.CWFOFNyA1N /tmp/tmp.atw4LTrZWw
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.A46yPzwOJi
++ mktemp
+ local LAST_ERR=/tmp/tmp.3tM8MOBA8i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A46yPzwOJi
+ cat /tmp/tmp.3tM8MOBA8i
+ rm /tmp/tmp.A46yPzwOJi /tmp/tmp.3tM8MOBA8i
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2LHCki2JP7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VI5PnXhyFu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.byE9Gr5aIB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VI5PnXhyFu
++ cat /tmp/tmp.byE9Gr5aIB
++ rm /tmp/tmp.VI5PnXhyFu /tmp/tmp.byE9Gr5aIB
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SMBHC3imP4
++ mktemp
+ local LAST_ERR=/tmp/tmp.bVO5WgDpHB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SMBHC3imP4
+ cat /tmp/tmp.bVO5WgDpHB
+ rm /tmp/tmp.SMBHC3imP4 /tmp/tmp.bVO5WgDpHB
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2LHCki2JP7/find2
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest backup-gcp-cs
++ local backup_name=backup-gcp-cs
++ kubectl_bin get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C415TwBBvO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jdh9zX7lxj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C415TwBBvO
++ cat /tmp/tmp.jdh9zX7lxj
++ rm /tmp/tmp.C415TwBBvO /tmp/tmp.jdh9zX7lxj
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z
+ gunzip
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z/rs0/myApp.test.gz
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z/rs1/myApp1.test.gz
+ gunzip
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z/rs2/myApp2.test.gz
+ gunzip
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XOY2nrmHvV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zPzDfvDNDj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XOY2nrmHvV
++ cat /tmp/tmp.zPzDfvDNDj
++ rm /tmp/tmp.XOY2nrmHvV /tmp/tmp.zPzDfvDNDj
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0WLIWHrU3j
++ mktemp
+ local LAST_ERR=/tmp/tmp.vp4aiomABV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0WLIWHrU3j
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("18651062-8537-43dd-9aa9-15dfe9d5a88e") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.vp4aiomABV
+ rm /tmp/tmp.0WLIWHrU3j /tmp/tmp.vp4aiomABV
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NabriwEPh3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LUVG1HM2Kn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NabriwEPh3
++ cat /tmp/tmp.LUVG1HM2Kn
++ rm /tmp/tmp.NabriwEPh3 /tmp/tmp.LUVG1HM2Kn
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2MOtmZNHs4
++ mktemp
+ local LAST_ERR=/tmp/tmp.uKl2ejPGQR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2MOtmZNHs4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5ec87d08-c598-4a96-8e59-e47f553a8df8") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.uKl2ejPGQR
+ rm /tmp/tmp.2MOtmZNHs4 /tmp/tmp.uKl2ejPGQR
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUL9KLFyD7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cg9m5zlg6y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gUL9KLFyD7
++ cat /tmp/tmp.cg9m5zlg6y
++ rm /tmp/tmp.gUL9KLFyD7 /tmp/tmp.cg9m5zlg6y
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.X8ccfZCksz
++ mktemp
+ local LAST_ERR=/tmp/tmp.raEnPqz4Tt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X8ccfZCksz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("332d8482-adbf-4378-9d65-0c223dfb78df") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.raEnPqz4Tt
+ rm /tmp/tmp.X8ccfZCksz /tmp/tmp.raEnPqz4Tt
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
egrep: warning: egrep is obsolescent; using grep -E
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MOSQ110tHJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R1V7xSa82S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MOSQ110tHJ
++ cat /tmp/tmp.R1V7xSa82S
++ rm /tmp/tmp.MOSQ110tHJ /tmp/tmp.R1V7xSa82S
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kGxD5Vtz2A
++ mktemp
+ local LAST_ERR=/tmp/tmp.fxoVbMiCDE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kGxD5Vtz2A
+ cat /tmp/tmp.fxoVbMiCDE
+ rm /tmp/tmp.kGxD5Vtz2A /tmp/tmp.fxoVbMiCDE
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.2LHCki2JP7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LG9TtoP1KA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2MQsKUS0yQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LG9TtoP1KA
++ cat /tmp/tmp.2MQsKUS0yQ
++ rm /tmp/tmp.LG9TtoP1KA /tmp/tmp.2MQsKUS0yQ
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M0N0mEqVyU
++ mktemp
+ local LAST_ERR=/tmp/tmp.BAaAVnDAKf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M0N0mEqVyU
+ cat /tmp/tmp.BAaAVnDAKf
+ rm /tmp/tmp.M0N0mEqVyU /tmp/tmp.BAaAVnDAKf
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.2LHCki2JP7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mboSjE86JS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lP7V09KPST
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mboSjE86JS
++ cat /tmp/tmp.lP7V09KPST
++ rm /tmp/tmp.mboSjE86JS /tmp/tmp.lP7V09KPST
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dnJ5F46Rli
++ mktemp
+ local LAST_ERR=/tmp/tmp.O5mCibahr9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dnJ5F46Rli
+ cat /tmp/tmp.O5mCibahr9
+ rm /tmp/tmp.dnJ5F46Rli /tmp/tmp.O5mCibahr9
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.2LHCki2JP7/find-2nd2
+ run_restore backup-gcp-cs
+ local backup_name=backup-gcp-cs
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-gcp-cs/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-gcp-cs/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.UMHKxPVFAw
++ mktemp
+ local LAST_ERR=/tmp/tmp.VrRF9HRtOe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UMHKxPVFAw
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs created
+ cat /tmp/tmp.VrRF9HRtOe
+ rm /tmp/tmp.UMHKxPVFAw /tmp/tmp.VrRF9HRtOe
+ return 0
+ wait_restore backup-gcp-cs some-name
+ local backup_name=backup-gcp-cs
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs object to be created.OK
Waiting psmdb-restore/restore-backup-gcp-cs to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kswgq3rygI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UJEf9EXXvY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kswgq3rygI
++ cat /tmp/tmp.UJEf9EXXvY
++ rm /tmp/tmp.Kswgq3rygI /tmp/tmp.UJEf9EXXvY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fUZ4zWDLzg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.95MYDnCEBb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fUZ4zWDLzg
++ cat /tmp/tmp.95MYDnCEBb
++ rm /tmp/tmp.fUZ4zWDLzg /tmp/tmp.95MYDnCEBb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S6Z2UV6GeR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kXE3Le2otc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S6Z2UV6GeR
++ cat /tmp/tmp.kXE3Le2otc
++ rm /tmp/tmp.S6Z2UV6GeR /tmp/tmp.kXE3Le2otc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QNQN1ZD3CJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZoWNeKtjrz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QNQN1ZD3CJ
++ cat /tmp/tmp.ZoWNeKtjrz
++ rm /tmp/tmp.QNQN1ZD3CJ /tmp/tmp.ZoWNeKtjrz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0DfV6XVhkM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RPrwyt6AoQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0DfV6XVhkM
++ cat /tmp/tmp.RPrwyt6AoQ
++ rm /tmp/tmp.0DfV6XVhkM /tmp/tmp.RPrwyt6AoQ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KtjGroxoJO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tUQeIFX4C8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KtjGroxoJO
++ cat /tmp/tmp.tUQeIFX4C8
++ rm /tmp/tmp.KtjGroxoJO /tmp/tmp.tUQeIFX4C8
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kXPmjxE629
++ mktemp
+ local LAST_ERR=/tmp/tmp.Z3jP7Ijc6M
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kXPmjxE629
+ cat /tmp/tmp.Z3jP7Ijc6M
+ rm /tmp/tmp.kXPmjxE629 /tmp/tmp.Z3jP7Ijc6M
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2LHCki2JP7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZkiIBGDStT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Y7eSYPcARN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZkiIBGDStT
++ cat /tmp/tmp.Y7eSYPcARN
++ rm /tmp/tmp.ZkiIBGDStT /tmp/tmp.Y7eSYPcARN
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TU9hVuF2kX
++ mktemp
+ local LAST_ERR=/tmp/tmp.LVmu6pvkuk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TU9hVuF2kX
+ cat /tmp/tmp.LVmu6pvkuk
+ rm /tmp/tmp.TU9hVuF2kX /tmp/tmp.LVmu6pvkuk
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2LHCki2JP7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pPpVEb1Zuy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rC8WSs2QEs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pPpVEb1Zuy
++ cat /tmp/tmp.rC8WSs2QEs
++ rm /tmp/tmp.pPpVEb1Zuy /tmp/tmp.rC8WSs2QEs
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NFyAofLFsp
++ mktemp
+ local LAST_ERR=/tmp/tmp.GTbmo78NfD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NFyAofLFsp
+ cat /tmp/tmp.GTbmo78NfD
+ rm /tmp/tmp.NFyAofLFsp /tmp/tmp.GTbmo78NfD
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2LHCki2JP7/find2
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest backup-azure-blob
++ local backup_name=backup-azure-blob
++ kubectl_bin get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.dlGCuDA9Pm
++ sed 's|gs://||'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C7axmeMkaa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-azure-blob -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dlGCuDA9Pm
++ cat /tmp/tmp.C7axmeMkaa
++ rm /tmp/tmp.dlGCuDA9Pm /tmp/tmp.C7axmeMkaa
++ return 0
+ backup_dest_azure=operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z/rs0/myApp.test.gz
+ gunzip
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z/rs1/myApp1.test.gz
+ gunzip
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z/rs2/myApp2.test.gz
+ gunzip
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0b3Yr5kiuY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uFDEaO2soG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0b3Yr5kiuY
++ cat /tmp/tmp.uFDEaO2soG
++ rm /tmp/tmp.0b3Yr5kiuY /tmp/tmp.uFDEaO2soG
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DfPghoGD0K
++ mktemp
+ local LAST_ERR=/tmp/tmp.NJ03sdDBwr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DfPghoGD0K
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("60028bf9-9fbd-4303-ad19-b093d08064a6") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.NJ03sdDBwr
+ rm /tmp/tmp.DfPghoGD0K /tmp/tmp.NJ03sdDBwr
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q4OwIxmGGB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9AIjHALFfz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q4OwIxmGGB
++ cat /tmp/tmp.9AIjHALFfz
++ rm /tmp/tmp.Q4OwIxmGGB /tmp/tmp.9AIjHALFfz
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.x2l69fEuT2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Noox5flHf1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.x2l69fEuT2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("ef9fd969-1fec-44c9-9962-bc9e8e2755d8") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Noox5flHf1
+ rm /tmp/tmp.x2l69fEuT2 /tmp/tmp.Noox5flHf1
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.retDLEejc2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sUY7ZvLu5F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.retDLEejc2
++ cat /tmp/tmp.sUY7ZvLu5F
++ rm /tmp/tmp.retDLEejc2 /tmp/tmp.sUY7ZvLu5F
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EIvRPNR5ee
++ mktemp
+ local LAST_ERR=/tmp/tmp.kClNUGiASj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EIvRPNR5ee
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6a675678-da02-450c-80fc-f5954a498f8f") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.kClNUGiASj
+ rm /tmp/tmp.EIvRPNR5ee /tmp/tmp.kClNUGiASj
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1mmTwPcMn2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.huLLdAaJee
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1mmTwPcMn2
++ cat /tmp/tmp.huLLdAaJee
++ rm /tmp/tmp.1mmTwPcMn2 /tmp/tmp.huLLdAaJee
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qmaNvt1icx
++ mktemp
+ local LAST_ERR=/tmp/tmp.bD9nA5Qu81
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qmaNvt1icx
+ cat /tmp/tmp.bD9nA5Qu81
+ rm /tmp/tmp.qmaNvt1icx /tmp/tmp.bD9nA5Qu81
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.2LHCki2JP7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JjCmRxOjDS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h90yHixw1v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JjCmRxOjDS
++ cat /tmp/tmp.h90yHixw1v
++ rm /tmp/tmp.JjCmRxOjDS /tmp/tmp.h90yHixw1v
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.taHDqj79BN
++ mktemp
+ local LAST_ERR=/tmp/tmp.rL0IW25b9h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.taHDqj79BN
+ cat /tmp/tmp.rL0IW25b9h
+ rm /tmp/tmp.taHDqj79BN /tmp/tmp.rL0IW25b9h
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.2LHCki2JP7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V94PpYZ9wD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K5wk0hA1RX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V94PpYZ9wD
++ cat /tmp/tmp.K5wk0hA1RX
++ rm /tmp/tmp.V94PpYZ9wD /tmp/tmp.K5wk0hA1RX
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FtsPVHas9C
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZYLDRYZnQq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FtsPVHas9C
+ cat /tmp/tmp.ZYLDRYZnQq
+ rm /tmp/tmp.FtsPVHas9C /tmp/tmp.ZYLDRYZnQq
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.2LHCki2JP7/find-2nd2
+ run_restore backup-azure-blob
+ local backup_name=backup-azure-blob
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-azure-blob/'
+ /usr/sbin/sed -e 's/name:/name: restore-backup-azure-blob/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.N5IBMnxMVF
++ mktemp
+ local LAST_ERR=/tmp/tmp.oFDGZqCNJV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N5IBMnxMVF
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob created
+ cat /tmp/tmp.oFDGZqCNJV
+ rm /tmp/tmp.N5IBMnxMVF /tmp/tmp.oFDGZqCNJV
+ return 0
+ wait_restore backup-azure-blob some-name
+ local backup_name=backup-azure-blob
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob object to be created.OK
Waiting psmdb-restore/restore-backup-azure-blob to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B5eAZi1kuR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cFgBkDGwj5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B5eAZi1kuR
++ cat /tmp/tmp.cFgBkDGwj5
++ rm /tmp/tmp.B5eAZi1kuR /tmp/tmp.cFgBkDGwj5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eeOn8BbD88
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iFYdHglES5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eeOn8BbD88
++ cat /tmp/tmp.iFYdHglES5
++ rm /tmp/tmp.eeOn8BbD88 /tmp/tmp.iFYdHglES5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dac6JUDNCX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aOmIdwYpn6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dac6JUDNCX
++ cat /tmp/tmp.aOmIdwYpn6
++ rm /tmp/tmp.Dac6JUDNCX /tmp/tmp.aOmIdwYpn6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9E1jM6iiI4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gm9ikdZpOX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9E1jM6iiI4
++ cat /tmp/tmp.gm9ikdZpOX
++ rm /tmp/tmp.9E1jM6iiI4 /tmp/tmp.gm9ikdZpOX
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xeco6CAxyG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XwvRreZbUG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xeco6CAxyG
++ cat /tmp/tmp.XwvRreZbUG
++ rm /tmp/tmp.Xeco6CAxyG /tmp/tmp.XwvRreZbUG
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rry8GFLZIO
++ mktemp
+ local LAST_ERR=/tmp/tmp.U8blI3viVY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Rry8GFLZIO
+ cat /tmp/tmp.U8blI3viVY
+ rm /tmp/tmp.Rry8GFLZIO /tmp/tmp.U8blI3viVY
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2LHCki2JP7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fderizo4lv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WTm4OurJfb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fderizo4lv
++ cat /tmp/tmp.WTm4OurJfb
++ rm /tmp/tmp.fderizo4lv /tmp/tmp.WTm4OurJfb
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XXDl18kDy8
++ mktemp
+ local LAST_ERR=/tmp/tmp.8yGhofUB9T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XXDl18kDy8
+ cat /tmp/tmp.8yGhofUB9T
+ rm /tmp/tmp.XXDl18kDy8 /tmp/tmp.8yGhofUB9T
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2LHCki2JP7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.23zCEFvdod
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z4Da3mFTj7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.23zCEFvdod
++ cat /tmp/tmp.Z4Da3mFTj7
++ rm /tmp/tmp.23zCEFvdod /tmp/tmp.Z4Da3mFTj7
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pbFlQv4x09
++ mktemp
+ local LAST_ERR=/tmp/tmp.MfLwufLo1s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pbFlQv4x09
+ cat /tmp/tmp.MfLwufLo1s
+ rm /tmp/tmp.pbFlQv4x09 /tmp/tmp.MfLwufLo1s
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2LHCki2JP7/find2
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ check_backup_in_storage backup-minio minio rs0 myApp.test.gz
+ local backup=backup-minio
+ local storage_type=minio
+ local replset=rs0
+ local file=myApp.test.gz
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio
++ local backup_name=backup-minio
++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed 's|gs://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7QLVoSw1Ng
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QbXOBWAxQG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup backup-minio -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7QLVoSw1Ng
++ cat /tmp/tmp.QbXOBWAxQG
++ rm /tmp/tmp.7QLVoSw1Ng /tmp/tmp.QbXOBWAxQG
++ return 0
+ backup_dest=operator-testing/2025-10-17T20:31:31Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-17T20:31:31Z/rs0/myApp.test.gz
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.LOilTUjM3R
++ mktemp
+ local LAST_ERR=/tmp/tmp.8Bk24cCBpc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-10-17T20:31:31Z/rs0/myApp.test.gz
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LOilTUjM3R
+ cat /tmp/tmp.8Bk24cCBpc
+ rm /tmp/tmp.LOilTUjM3R /tmp/tmp.8Bk24cCBpc
+ return 0
2025-10-17 20:31:35         55 myApp.test.gz
+ insert_data_mongos 100501 myApp '' 27019
+ local data=100501
+ local db_name=myApp
+ local flags=
+ local port=27019
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yl5h3pCxkl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iWBoUXnCAt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Yl5h3pCxkl
++ cat /tmp/tmp.iWBoUXnCAt
++ rm /tmp/tmp.Yl5h3pCxkl /tmp/tmp.iWBoUXnCAt
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0yUAVM3PY6
++ mktemp
+ local LAST_ERR=/tmp/tmp.mI3TxPXsrV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0yUAVM3PY6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("94d032d4-a669-4edc-80de-edb1dfe49803") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mI3TxPXsrV
+ rm /tmp/tmp.0yUAVM3PY6 /tmp/tmp.mI3TxPXsrV
+ return 0
+ insert_data_mongos 100501 myApp1 '' 27019
+ local data=100501
+ local db_name=myApp1
+ local flags=
+ local port=27019
+ run_mongos 'use myApp1\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp1\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.40bfdCDD0f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jIGKhRfnCU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.40bfdCDD0f
++ cat /tmp/tmp.jIGKhRfnCU
++ rm /tmp/tmp.40bfdCDD0f /tmp/tmp.jIGKhRfnCU
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.bZDtkxExSC
++ mktemp
+ local LAST_ERR=/tmp/tmp.kO1AhsR2IS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bZDtkxExSC
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("4fe6ad0b-07be-4fb4-bac0-90d2a62098f2") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp1
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.kO1AhsR2IS
+ rm /tmp/tmp.bZDtkxExSC /tmp/tmp.kO1AhsR2IS
+ return 0
+ insert_data_mongos 100501 myApp2 '' 27019
+ local data=100501
+ local db_name=myApp2
+ local flags=
+ local port=27019
+ run_mongos 'use myApp2\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' '' '' 27019
+ local 'command=use myApp2\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6G4ceYGsCW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0iLXZ5UBVd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6G4ceYGsCW
++ cat /tmp/tmp.0iLXZ5UBVd
++ rm /tmp/tmp.6G4ceYGsCW /tmp/tmp.0iLXZ5UBVd
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pAjGweQSMl
++ mktemp
+ local LAST_ERR=/tmp/tmp.OnJuhnXaO4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pAjGweQSMl
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("67b11d1d-af41-46f9-bd4d-32afebc0c0d6") }
Percona Server for MongoDB server version: v7.0.24-13
WARNING: shell and server versions do not match
switched to db myApp2
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.OnJuhnXaO4
+ rm /tmp/tmp.pAjGweQSMl /tmp/tmp.OnJuhnXaO4
+ return 0
+ check_data -2nd
+ local postfix=-2nd
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zD72PXHtCq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FzFzPKLCuy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zD72PXHtCq
++ cat /tmp/tmp.FzFzPKLCuy
++ rm /tmp/tmp.zD72PXHtCq /tmp/tmp.FzFzPKLCuy
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xUlWLownuQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.zdlaVBSlI5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xUlWLownuQ
+ cat /tmp/tmp.zdlaVBSlI5
+ rm /tmp/tmp.xUlWLownuQ /tmp/tmp.zdlaVBSlI5
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd.json /tmp/tmp.2LHCki2JP7/find-2nd
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dSIUkyiySr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mS2YFeBYRF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dSIUkyiySr
++ cat /tmp/tmp.mS2YFeBYRF
++ rm /tmp/tmp.dSIUkyiySr /tmp/tmp.mS2YFeBYRF
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4hlltSN9T4
++ mktemp
+ local LAST_ERR=/tmp/tmp.TUJbBLDA9h
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4hlltSN9T4
+ cat /tmp/tmp.TUJbBLDA9h
+ rm /tmp/tmp.4hlltSN9T4 /tmp/tmp.TUJbBLDA9h
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd1.json /tmp/tmp.2LHCki2JP7/find-2nd1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 -2nd2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=-2nd2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G7svvOu8bg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iMg3J2KUmO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G7svvOu8bg
++ cat /tmp/tmp.iMg3J2KUmO
++ rm /tmp/tmp.G7svvOu8bg /tmp/tmp.iMg3J2KUmO
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IGO0XdLB9X
++ mktemp
+ local LAST_ERR=/tmp/tmp.22aZErHYp7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IGO0XdLB9X
+ cat /tmp/tmp.22aZErHYp7
+ rm /tmp/tmp.IGO0XdLB9X /tmp/tmp.22aZErHYp7
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find-2nd2.json /tmp/tmp.2LHCki2JP7/find-2nd2
+ run_restore backup-minio
+ local backup_name=backup-minio
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/conf/restore.yml
+ /usr/sbin/sed -e 's/name:/name: restore-backup-minio/'
+ /usr/sbin/sed -e 's/backupName:/backupName: backup-minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.wxsW11t79O
++ mktemp
+ local LAST_ERR=/tmp/tmp.M7RE6DLFpu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wxsW11t79O
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio created
+ cat /tmp/tmp.M7RE6DLFpu
+ rm /tmp/tmp.wxsW11t79O /tmp/tmp.M7RE6DLFpu
+ return 0
+ wait_restore backup-minio some-name
+ local backup_name=backup-minio
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio object to be created.OK
Waiting psmdb-restore/restore-backup-minio to reach state "ready" ..OK after 1 minutes
+ [[ 1 -eq 1 ]]
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fW5Wmm5Wbw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IchIXPJQpE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fW5Wmm5Wbw
++ cat /tmp/tmp.IchIXPJQpE
++ rm /tmp/tmp.fW5Wmm5Wbw /tmp/tmp.IchIXPJQpE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6ubs3nilj4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LH7DCoZfzo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6ubs3nilj4
++ cat /tmp/tmp.LH7DCoZfzo
++ rm /tmp/tmp.6ubs3nilj4 /tmp/tmp.LH7DCoZfzo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pWTFoPEflY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dnFrJXDL5z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pWTFoPEflY
++ cat /tmp/tmp.dnFrJXDL5z
++ rm /tmp/tmp.pWTFoPEflY /tmp/tmp.dnFrJXDL5z
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dYzBS4pDtz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FpClcpBCUU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dYzBS4pDtz
++ cat /tmp/tmp.FpClcpBCUU
++ rm /tmp/tmp.dYzBS4pDtz /tmp/tmp.FpClcpBCUU
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ check_data
+ local postfix=
++ seq 0 2
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 '' .svc.cluster.local myApp test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=
+ local suffix=.svc.cluster.local
+ local database=myApp
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fh4fG2GjnM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fc1KwEJxWI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Fh4fG2GjnM
++ cat /tmp/tmp.fc1KwEJxWI
++ rm /tmp/tmp.Fh4fG2GjnM /tmp/tmp.fc1KwEJxWI
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dslWRXhQpN
++ mktemp
+ local LAST_ERR=/tmp/tmp.pBBQoFBybu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dslWRXhQpN
+ cat /tmp/tmp.pBBQoFBybu
+ rm /tmp/tmp.dslWRXhQpN /tmp/tmp.pBBQoFBybu
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.2LHCki2JP7/find
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 1 .svc.cluster.local myApp1 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=1
+ local suffix=.svc.cluster.local
+ local database=myApp1
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp1\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ local 'command=use myApp1\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
egrep: warning: egrep is obsolescent; using grep -E
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Q54VwWPey
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j7lh5gQxWm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Q54VwWPey
++ cat /tmp/tmp.j7lh5gQxWm
++ rm /tmp/tmp.5Q54VwWPey /tmp/tmp.j7lh5gQxWm
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MNgEFdzP4S
++ mktemp
+ local LAST_ERR=/tmp/tmp.B2lFWNUwCH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp1\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MNgEFdzP4S
+ cat /tmp/tmp.B2lFWNUwCH
+ rm /tmp/tmp.MNgEFdzP4S /tmp/tmp.B2lFWNUwCH
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find1.json /tmp/tmp.2LHCki2JP7/find1
+ for i in $(seq 0 2)
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-15678 2 .svc.cluster.local myApp2 test 27019
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local postfix=2
+ local suffix=.svc.cluster.local
+ local database=myApp2
+ local collection=test
+ local port=27019
+ local tls=false
+ [[ false == \t\r\u\e ]]
+ mongos_command=run_mongos
+ run_mongos 'use myApp2\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-15678 mongodb .svc.cluster.local '' 27019
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service'
+ local 'command=use myApp2\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-15678
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27019
+ local mongo_bin=mongo
+ /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
egrep: warning: egrep is obsolescent; using grep -E
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27019
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zXgvbWcYqg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qX5uEVSUMU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zXgvbWcYqg
++ cat /tmp/tmp.qX5uEVSUMU
++ rm /tmp/tmp.zXgvbWcYqg /tmp/tmp.qX5uEVSUMU
++ return 0
+ local client_container=psmdb-client-696897d69b-x8gtj
+ kubectl_bin exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ckSryv7313
++ mktemp
+ local LAST_ERR=/tmp/tmp.SbwKzxRGVJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-x8gtj -- bash -c 'printf '\''use myApp2\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-15678.svc.cluster.local:27019/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ckSryv7313
+ cat /tmp/tmp.SbwKzxRGVJ
+ rm /tmp/tmp.ckSryv7313 /tmp/tmp.SbwKzxRGVJ
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/demand-backup-sharded/compare/find2.json /tmp/tmp.2LHCki2JP7/find2
+ desc 'delete backup and check if it is removed from bucket -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete backup and check if it is removed from bucket -- minio
-----------------------------------------------------------------------------------

+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.CYvc001XGG
++ mktemp
+ local LAST_ERR=/tmp/tmp.IbTKXJWdV5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CYvc001XGG
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3" deleted from demand-backup-sharded-15678 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob" deleted from demand-backup-sharded-15678 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs" deleted from demand-backup-sharded-15678 namespace
perconaservermongodbbackup.psmdb.percona.com "backup-minio" deleted from demand-backup-sharded-15678 namespace
+ cat /tmp/tmp.IbTKXJWdV5
+ rm /tmp/tmp.CYvc001XGG /tmp/tmp.IbTKXJWdV5
+ return 0
++ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ grep -c _
++ cat
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YygWnEDLzs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c0zCmJhp51
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YygWnEDLzs
++ cat /tmp/tmp.c0zCmJhp51
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-sharded-15678
++ rm /tmp/tmp.YygWnEDLzs /tmp/tmp.c0zCmJhp51
++ return 0
+ backup_exists=0
+ [[ 0 -eq 1 ]]
+ '[' -z '' ']'
+ check_backup_deletion https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z aws-s3
+ local path=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z
+ local storage_name=aws-s3
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:36Z
+ [[ 403 -eq 403 ]]
+ check_backup_deletion https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z gcp-cs
+ local path=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z
+ local storage_name=gcp-cs
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://storage.googleapis.com/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:33:09Z
+ [[ 404 -eq 404 ]]
+ check_backup_deletion https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z azure-blob
+ local path=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z
+ local storage_name=azure-blob
+ local retry=0
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z
+ [[ 404 -eq 403 ]]
++ curl -sw '%{http_code}' -o /dev/null https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-sharded/2025-10-17T20:32:03Z
+ [[ 404 -eq 404 ]]
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/container-rc.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SNvbnCArkA
++ mktemp
+ local LAST_ERR=/tmp/tmp.KPGiwjPb1B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/e2e-tests/conf/container-rc.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SNvbnCArkA
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.KPGiwjPb1B
+ rm /tmp/tmp.SNvbnCArkA /tmp/tmp.KPGiwjPb1B
+ return 0
+ destroy demand-backup-sharded-15678
+ local namespace=demand-backup-sharded-15678
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.djHbHfTshs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HdyPx2qw5Q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.djHbHfTshs
++ cat /tmp/tmp.HdyPx2qw5Q
No resources found in demand-backup-sharded-15678 namespace.
++ rm /tmp/tmp.djHbHfTshs /tmp/tmp.HdyPx2qw5Q
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.gOwwxPhnZA
++ mktemp
+ local LAST_ERR=/tmp/tmp.SCA8Y3Ltzl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gOwwxPhnZA
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.SCA8Y3Ltzl
+ rm /tmp/tmp.gOwwxPhnZA /tmp/tmp.SCA8Y3Ltzl
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.pcKQ3buNwB
++ mktemp
+ local LAST_ERR=/tmp/tmp.gyrwCeldof
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pcKQ3buNwB
+ cat /tmp/tmp.gyrwCeldof
+ rm /tmp/tmp.pcKQ3buNwB /tmp/tmp.gyrwCeldof
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.74zto3mwzQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.z369P15r8I
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.74zto3mwzQ
+ cat /tmp/tmp.z369P15r8I
+ rm /tmp/tmp.74zto3mwzQ /tmp/tmp.z369P15r8I
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3qnkFOyGP8
++ mktemp
+ local LAST_ERR=/tmp/tmp.6pn90Q0vX3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3qnkFOyGP8
+ cat /tmp/tmp.6pn90Q0vX3
+ rm /tmp/tmp.3qnkFOyGP8 /tmp/tmp.6pn90Q0vX3
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.xGhjSEZABH
++ mktemp
+ local LAST_ERR=/tmp/tmp.l2p1hAneMD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2094/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xGhjSEZABH
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.l2p1hAneMD
+ rm /tmp/tmp.xGhjSEZABH /tmp/tmp.l2p1hAneMD
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bFAq3ZnMu4
++ mktemp
+ local LAST_ERR=/tmp/tmp.puIbRIZu53
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bFAq3ZnMu4
+ cat /tmp/tmp.puIbRIZu53
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bFAq3ZnMu4
+ cat /tmp/tmp.puIbRIZu53
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.bFAq3ZnMu4
+ cat /tmp/tmp.puIbRIZu53
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.bFAq3ZnMu4
+ cat /tmp/tmp.puIbRIZu53
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.bFAq3ZnMu4 /tmp/tmp.puIbRIZu53
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-sharded-15678
+ rm -rf /tmp/tmp.2LHCki2JP7
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.T7e175Yms0
++ mktemp
+ local LAST_OUT=/tmp/tmp.s6DUGCZjmC
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kq8xmfmsoV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.d2zybuwX23
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-sharded-15678