Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/logs/demand-backup-physical-sharded.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-physical-sharded-8499
+ local ns=demand-backup-physical-sharded-8499
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.FYkJYfFTqp
++ mktemp
+ local LAST_ERR=/tmp/tmp.PqEUjHnPq6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FYkJYfFTqp
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.PqEUjHnPq6
+ rm /tmp/tmp.FYkJYfFTqp /tmp/tmp.PqEUjHnPq6
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-11612 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-aws-s3-sharded" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-11612 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-azure-blob-sharded" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-11612 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-11612 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.D4yWViYr0U
++ mktemp
+ local LAST_ERR=/tmp/tmp.AsDyckF2kU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D4yWViYr0U
+ cat /tmp/tmp.AsDyckF2kU
+ rm /tmp/tmp.D4yWViYr0U /tmp/tmp.AsDyckF2kU
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.HlMsqV95Q3
++ mktemp
+ local LAST_ERR=/tmp/tmp.nEepXGEiMq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HlMsqV95Q3
+ cat /tmp/tmp.nEepXGEiMq
+ rm /tmp/tmp.HlMsqV95Q3 /tmp/tmp.nEepXGEiMq
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.3EzyLlhW5r
++ mktemp
+ local LAST_ERR=/tmp/tmp.rfAKUpDUiU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3EzyLlhW5r
+ cat /tmp/tmp.rfAKUpDUiU
+ rm /tmp/tmp.3EzyLlhW5r /tmp/tmp.rfAKUpDUiU
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.FuhpqLlw4X
++ mktemp
+ local LAST_ERR=/tmp/tmp.BgAcChpZ9z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FuhpqLlw4X
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.BgAcChpZ9z
+ rm /tmp/tmp.FuhpqLlw4X /tmp/tmp.BgAcChpZ9z
+ return 0
+ check_crd_for_deletion PR-1915-2d829121
+ local git_tag=PR-1915-2d829121
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1915-2d829121/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NrnnssifIg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WxYSSuhaWK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NrnnssifIg
++ cat /tmp/tmp.WxYSSuhaWK
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NrnnssifIg
++ cat /tmp/tmp.WxYSSuhaWK
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.NrnnssifIg
++ cat /tmp/tmp.WxYSSuhaWK
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.NrnnssifIg
++ cat /tmp/tmp.WxYSSuhaWK
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.NrnnssifIg /tmp/tmp.WxYSSuhaWK
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ kubectl_bin get ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.KCtVipVUU4
+ local LAST_OUT=/tmp/tmp.N9sqr5c6sj
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.IrCngsJjpt
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.M7nLJwd1Gi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N9sqr5c6sj
+ cat /tmp/tmp.IrCngsJjpt
+ rm /tmp/tmp.N9sqr5c6sj /tmp/tmp.IrCngsJjpt
+ return 0
namespace "demand-backup-physical-sharded-11612" deleted
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KCtVipVUU4
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.M7nLJwd1Gi
+ rm /tmp/tmp.KCtVipVUU4 /tmp/tmp.M7nLJwd1Gi
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.VoS0u4o1DA
++ mktemp
+ local LAST_ERR=/tmp/tmp.XNrK2J8hRv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VoS0u4o1DA
+ cat /tmp/tmp.XNrK2J8hRv
+ rm /tmp/tmp.VoS0u4o1DA /tmp/tmp.XNrK2J8hRv
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.q50TSLYIBF
++ mktemp
+ local LAST_ERR=/tmp/tmp.fvV7cCWB5i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q50TSLYIBF
namespace/psmdb-operator created
+ cat /tmp/tmp.fvV7cCWB5i
+ rm /tmp/tmp.q50TSLYIBF /tmp/tmp.fvV7cCWB5i
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IbiPi7d3BJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G4G9Ujwt2V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IbiPi7d3BJ
++ cat /tmp/tmp.G4G9Ujwt2V
++ rm /tmp/tmp.IbiPi7d3BJ /tmp/tmp.G4G9Ujwt2V
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.JtgWsGPPFf
++ mktemp
+ local LAST_ERR=/tmp/tmp.NdbUa2LhLU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JtgWsGPPFf
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5" modified.
+ cat /tmp/tmp.NdbUa2LhLU
+ rm /tmp/tmp.JtgWsGPPFf /tmp/tmp.NdbUa2LhLU
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.C1pDAEX1Fy
++ mktemp
+ local LAST_ERR=/tmp/tmp.q2VGg0mrmA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C1pDAEX1Fy
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.q2VGg0mrmA
+ rm /tmp/tmp.C1pDAEX1Fy /tmp/tmp.q2VGg0mrmA
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.y5ZodBfDqc
++ mktemp
+ local LAST_ERR=/tmp/tmp.yEnFMN5UbM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.y5ZodBfDqc
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.yEnFMN5UbM
+ rm /tmp/tmp.y5ZodBfDqc /tmp/tmp.yEnFMN5UbM
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.H9BvV6plEq
++ mktemp
+ local LAST_ERR=/tmp/tmp.o9cPYMeJJl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H9BvV6plEq
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.o9cPYMeJJl
+ rm /tmp/tmp.H9BvV6plEq /tmp/tmp.o9cPYMeJJl
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cx8ASEC33a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PGjaGwI0sm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cx8ASEC33a
++ cat /tmp/tmp.PGjaGwI0sm
++ rm /tmp/tmp.cx8ASEC33a /tmp/tmp.PGjaGwI0sm
++ return 0
+ wait_pod percona-server-mongodb-operator-6c96ccdc79-47clg
+ local pod=percona-server-mongodb-operator-6c96ccdc79-47clg
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-6c96ccdc79-47clg to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rw1doi3VGi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JOy0qy4tSk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rw1doi3VGi
++ cat /tmp/tmp.JOy0qy4tSk
++ rm /tmp/tmp.rw1doi3VGi /tmp/tmp.JOy0qy4tSk
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-6c96ccdc79-47clg
++ mktemp
+ local LAST_OUT=/tmp/tmp.GYMuejHl7F
++ mktemp
+ local LAST_ERR=/tmp/tmp.vCarelIF24
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-6c96ccdc79-47clg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GYMuejHl7F
+ cat /tmp/tmp.vCarelIF24
+ rm /tmp/tmp.GYMuejHl7F /tmp/tmp.vCarelIF24
+ return 0
2025-05-21T09:41:07.641Z	INFO	setup	Manager starting up	{"gitCommit": "2d8291219bbb1a7ad015786c5ebd76d37f8fba63", "gitBranch": "PR-1915-2d829121", "buildTime": "", "goVersion": "go1.24.3", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-physical-sharded-8499
+ local namespace=demand-backup-physical-sharded-8499
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-physical-sharded-8499'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-sharded-8499
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-physical-sharded-8499 --ignore-not-found
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.mcKYeQCK4B
++ mktemp
+ local LAST_OUT=/tmp/tmp.h5IDZSH96t
++ mktemp
+ local LAST_ERR=/tmp/tmp.KKj8Ti6BV3
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.rIInzNAZIG
++ seq 0 2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-physical-sharded-8499 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mcKYeQCK4B
+ cat /tmp/tmp.KKj8Ti6BV3
+ rm /tmp/tmp.mcKYeQCK4B /tmp/tmp.KKj8Ti6BV3
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h5IDZSH96t
+ cat /tmp/tmp.rIInzNAZIG
+ rm /tmp/tmp.h5IDZSH96t /tmp/tmp.rIInzNAZIG
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-8499
++ mktemp
+ local LAST_OUT=/tmp/tmp.6KxxGtj6Pu
++ mktemp
+ local LAST_ERR=/tmp/tmp.f1NrZzfNtN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-sharded-8499
namespace "gke-managed-cim" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6KxxGtj6Pu
+ cat /tmp/tmp.f1NrZzfNtN
+ rm /tmp/tmp.6KxxGtj6Pu /tmp/tmp.f1NrZzfNtN
+ return 0
+ desc 'create namespace demand-backup-physical-sharded-8499'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-physical-sharded-8499
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-physical-sharded-8499
++ mktemp
+ local LAST_OUT=/tmp/tmp.FaQaGZVBqw
++ mktemp
+ local LAST_ERR=/tmp/tmp.keU8mM3P3J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-physical-sharded-8499
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FaQaGZVBqw
namespace/demand-backup-physical-sharded-8499 created
+ cat /tmp/tmp.keU8mM3P3J
+ rm /tmp/tmp.FaQaGZVBqw /tmp/tmp.keU8mM3P3J
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LkAvqdotcq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fcVCbbqGjV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LkAvqdotcq
++ cat /tmp/tmp.fcVCbbqGjV
++ rm /tmp/tmp.LkAvqdotcq /tmp/tmp.fcVCbbqGjV
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5 --namespace=demand-backup-physical-sharded-8499
++ mktemp
+ local LAST_OUT=/tmp/tmp.yC7fTiobLE
++ mktemp
+ local LAST_ERR=/tmp/tmp.NOEO3lqUdf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5 --namespace=demand-backup-physical-sharded-8499
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yC7fTiobLE
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1915-2d829121-4-cluster5" modified.
+ cat /tmp/tmp.NOEO3lqUdf
+ rm /tmp/tmp.yC7fTiobLE /tmp/tmp.NOEO3lqUdf
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed May 21 09:41:23 2025
NAMESPACE: demand-backup-physical-sharded-8499
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-physical-sharded-8499.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-sharded-8499 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-sharded-8499

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-sharded-8499 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-sharded-8499 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ONczSWLHIA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rmJhe6BOso
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ONczSWLHIA
++ cat /tmp/tmp.rmJhe6BOso
++ rm /tmp/tmp.ONczSWLHIA /tmp/tmp.rmJhe6BOso
++ return 0
+ MINIO_POD=minio-service-86dfccd949-msb2m
+ wait_pod minio-service-86dfccd949-msb2m
+ local pod=minio-service-86dfccd949-msb2m
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-msb2m to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-8499.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.KijNAPB1L2
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZAgh5PlO0T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-8499.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KijNAPB1L2
service/minio-service created
+ cat /tmp/tmp.ZAgh5PlO0T
+ rm /tmp/tmp.KijNAPB1L2 /tmp/tmp.ZAgh5PlO0T
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.fW8us6j0bZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.vdCdQ8EngO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fW8us6j0bZ
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.vdCdQ8EngO
+ rm /tmp/tmp.fW8us6j0bZ /tmp/tmp.vdCdQ8EngO
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.gYSTVt5Y4N
++ mktemp
+ local LAST_ERR=/tmp/tmp.IGoQoPs1TZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gYSTVt5Y4N
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.IGoQoPs1TZ
+ rm /tmp/tmp.gYSTVt5Y4N /tmp/tmp.IGoQoPs1TZ
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ echo 'Creating PSMDB cluster'
Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.R73wi13wGt
++ mktemp
+ local LAST_ERR=/tmp/tmp.PtpzdeN1wT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.R73wi13wGt
secret/some-users created
+ cat /tmp/tmp.PtpzdeN1wT
+ rm /tmp/tmp.R73wi13wGt /tmp/tmp.PtpzdeN1wT
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.rqGnZJaIFH
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ local LAST_ERR=/tmp/tmp.414Cs4T7OV
+ local exit_status=0
+ local timeout=4
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1915-2d829121"'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rqGnZJaIFH
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.414Cs4T7OV
+ rm /tmp/tmp.rqGnZJaIFH /tmp/tmp.414Cs4T7OV
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bha3ZLJuNu
++ mktemp
+ local LAST_ERR=/tmp/tmp.jU8Ew3z9m1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bha3ZLJuNu
deployment.apps/psmdb-client created
+ cat /tmp/tmp.jU8Ew3z9m1
+ rm /tmp/tmp.bha3ZLJuNu /tmp/tmp.jU8Ew3z9m1
+ return 0
+ echo 'check if all pods started'
check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.du6pooHJ6H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sSGGAMLpGE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.du6pooHJ6H
++ cat /tmp/tmp.sSGGAMLpGE
++ rm /tmp/tmp.du6pooHJ6H /tmp/tmp.sSGGAMLpGE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.juq3uumVc9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6n5wutR4Lv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.juq3uumVc9
++ cat /tmp/tmp.6n5wutR4Lv
++ rm /tmp/tmp.juq3uumVc9 /tmp/tmp.6n5wutR4Lv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness...................................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0fD3fw29oQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2HNoqJ5TuL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0fD3fw29oQ
++ cat /tmp/tmp.2HNoqJ5TuL
++ rm /tmp/tmp.0fD3fw29oQ /tmp/tmp.2HNoqJ5TuL
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nAV6LEtFlf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9mlv7mmEZL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nAV6LEtFlf
++ cat /tmp/tmp.9mlv7mmEZL
++ rm /tmp/tmp.nAV6LEtFlf /tmp/tmp.9mlv7mmEZL
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FO8qL5B9lX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OYcxCWQMY9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FO8qL5B9lX
++ cat /tmp/tmp.OYcxCWQMY9
++ rm /tmp/tmp.FO8qL5B9lX /tmp/tmp.OYcxCWQMY9
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nbnCwx5vtM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i90uDoS9H8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nbnCwx5vtM
++ cat /tmp/tmp.i90uDoS9H8
++ rm /tmp/tmp.nbnCwx5vtM /tmp/tmp.i90uDoS9H8
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nLulin1Q4e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JPrkUo4i9L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nLulin1Q4e
++ cat /tmp/tmp.JPrkUo4i9L
++ rm /tmp/tmp.nLulin1Q4e /tmp/tmp.JPrkUo4i9L
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nGiN7YjOGP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yOMnl3cZsj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nGiN7YjOGP
++ cat /tmp/tmp.yOMnl3cZsj
++ rm /tmp/tmp.nGiN7YjOGP /tmp/tmp.yOMnl3cZsj
++ return 0
+ lbEndpoint=35.232.179.101
+ '[' -z 35.232.179.101 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O55RzwcjmY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4YN58tsq5I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O55RzwcjmY
++ cat /tmp/tmp.4YN58tsq5I
++ rm /tmp/tmp.O55RzwcjmY /tmp/tmp.4YN58tsq5I
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.scveY2gjh3
++ mktemp
+ local LAST_ERR=/tmp/tmp.8XdKazKPph
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.scveY2gjh3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f6df7391-177b-4a52-b5db-b90d6f3268f5") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.8XdKazKPph
+ rm /tmp/tmp.scveY2gjh3 /tmp/tmp.8XdKazKPph
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KMOA74bdNE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Iuaf7ZCE7i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KMOA74bdNE
++ cat /tmp/tmp.Iuaf7ZCE7i
++ rm /tmp/tmp.KMOA74bdNE /tmp/tmp.Iuaf7ZCE7i
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.RJ5wAASQNV
++ mktemp
+ local LAST_ERR=/tmp/tmp.5d2Pi7VBmy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RJ5wAASQNV
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("88eb9a6b-66b2-452b-93a3-b66865a6c258") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.5d2Pi7VBmy
+ rm /tmp/tmp.RJ5wAASQNV /tmp/tmp.5d2Pi7VBmy
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aKzilIuAIJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Bi0X7HgDMq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aKzilIuAIJ
++ cat /tmp/tmp.Bi0X7HgDMq
++ rm /tmp/tmp.aKzilIuAIJ /tmp/tmp.Bi0X7HgDMq
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hd0oMy5nlG
++ mktemp
+ local LAST_ERR=/tmp/tmp.93NJvXcQUB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hd0oMy5nlG
+ cat /tmp/tmp.93NJvXcQUB
+ rm /tmp/tmp.hd0oMy5nlG /tmp/tmp.93NJvXcQUB
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.6i402Be8ap/find-sharded
+ echo 'waiting 60 seconds for stable timestamp in wiredtiger'
waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ echo 'running backups'
running backups
+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-minio-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/storageName:/storageName: minio/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.tJfQdK5Esx
++ mktemp
+ local LAST_ERR=/tmp/tmp.chq1MxI1AW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tJfQdK5Esx
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.chq1MxI1AW
+ rm /tmp/tmp.tJfQdK5Esx /tmp/tmp.chq1MxI1AW
+ return 0
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: aws-s3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NDhl38adOc
++ mktemp
+ local LAST_ERR=/tmp/tmp.jl5eaItlRE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NDhl38adOc
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.jl5eaItlRE
+ rm /tmp/tmp.NDhl38adOc /tmp/tmp.jl5eaItlRE
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.EW9vT54DD5
++ mktemp
+ local LAST_ERR=/tmp/tmp.xdO1MoO0dk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EW9vT54DD5
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.xdO1MoO0dk
+ rm /tmp/tmp.EW9vT54DD5 /tmp/tmp.xdO1MoO0dk
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: azure-blob/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.suqF4BWkCH
++ mktemp
+ local LAST_ERR=/tmp/tmp.ayXKhZYzBT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.suqF4BWkCH
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.ayXKhZYzBT
+ rm /tmp/tmp.suqF4BWkCH /tmp/tmp.ayXKhZYzBT
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state...............................................................................
+ check_backup_in_storage backup-aws-s3-sharded s3 rs0
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ep9umimEcq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TzJD1DpdJd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ep9umimEcq
++ cat /tmp/tmp.TzJD1DpdJd
++ rm /tmp/tmp.ep9umimEcq /tmp/tmp.TzJD1DpdJd
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/rs0/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:22+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/rs0/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 12033    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: E3TVPl0k7M+LjwNC6LAKghq99AchCM8tRchhxpGmIgM5BkJkq9p+0hLh4jlGPzEfJ9+1DqFnjpg=
x-amz-request-id: 2GN3473NZ401Z4S9
Date: Wed, 21 May 2025 09:49:24 GMT
Last-Modified: Wed, 21 May 2025 09:49:14 GMT
x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "c261bb16b577cfabc6a0ad0bd8d331a1"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: binary/octet-stream
Content-Length: 12033
Server: AmazonS3

+ check_backup_in_storage backup-aws-s3-sharded s3 cfg
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.G4ipaYTXa1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9RWkhYaqw3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.G4ipaYTXa1
++ cat /tmp/tmp.9RWkhYaqw3
++ rm /tmp/tmp.G4ipaYTXa1 /tmp/tmp.9RWkhYaqw3
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/cfg/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:24+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/cfg/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:56Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18299    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: alP/C3IUhaYFqYto6XzOKCJs0cTod08drU/+2pgIiu/BoUldwBuopuH6RxNZbn49LNa6AHK4hfE=
x-amz-request-id: 8TBMGRRFN3W0WAEZ
Date: Wed, 21 May 2025 09:49:25 GMT
Last-Modified: Wed, 21 May 2025 09:49:19 GMT
x-amz-expiration: expiry-date="Fri, 23 May 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "cead34e6b26f5c4a694627c241f539da"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: binary/octet-stream
Content-Length: 18299
Server: AmazonS3

+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sharded to reach ready state.
+ check_backup_in_storage backup-gcp-cs-sharded gcs rs0
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.u4N8TQgdtN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YRcIU8NjLC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u4N8TQgdtN
++ cat /tmp/tmp.YRcIU8NjLC
++ rm /tmp/tmp.u4N8TQgdtN /tmp/tmp.YRcIU8NjLC
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:26+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 11959    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAO2Vwp2Dscxa3T3PfzcsAFRwOiZPv2BD0Ba1rcg1TJUmdr2y5uSEr_84PgkEBm3xoEZLMAkjGfPjz4
expires: Wed, 21 May 2025 10:49:27 GMT
date: Wed, 21 May 2025 09:49:27 GMT
cache-control: public, max-age=3600
last-modified: Wed, 21 May 2025 09:48:05 GMT
etag: "41fc5143e53aecbce91225ccaf8a6f3f"
x-goog-generation: 1747820885660092
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 11959
x-goog-hash: crc32c=smx4bg==
x-goog-hash: md5=QfxRQ+U67LzpEiXMr4pvPw==
x-goog-expiration: Thu, 22 May 2025 09:48:05 GMT
x-goog-storage-class: STANDARD
accept-ranges: bytes
content-length: 11959
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sharded gcs cfg
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kfMHdpn5qd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UWGmsMZ6dW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kfMHdpn5qd
++ cat /tmp/tmp.UWGmsMZ6dW
++ rm /tmp/tmp.kfMHdpn5qd /tmp/tmp.UWGmsMZ6dW
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:28+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:47:45Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18439    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18439    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: AAO2VwqGj9yEU1iZDvBqlA4h_hmsCQTdNjdcNXI-3hmTmJF5xU95Sv7JFUpi-seYCtyFXsDYVFK3k_o
expires: Wed, 21 May 2025 10:49:28 GMT
date: Wed, 21 May 2025 09:49:28 GMT
cache-control: public, max-age=3600
last-modified: Wed, 21 May 2025 09:48:05 GMT
etag: "5d2e39e5b33fede2dcd60b8a2fb5330f"
x-goog-generation: 1747820885337271
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 18439
x-goog-hash: crc32c=nu+dLw==
x-goog-hash: md5=XS455bM/7eLc1guKL7UzDw==
x-goog-expiration: Thu, 22 May 2025 09:48:05 GMT
x-goog-storage-class: STANDARD
accept-ranges: bytes
content-length: 18439
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob-sharded to reach ready state.
+ check_backup_in_storage backup-azure-blob-sharded azure rs0
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.4N58e5Pb2a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5qMnMAAwCK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4N58e5Pb2a
++ cat /tmp/tmp.5qMnMAAwCK
++ rm /tmp/tmp.4N58e5Pb2a /tmp/tmp.5qMnMAAwCK
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/rs0/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:30+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/rs0/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 12897    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 12897
Content-Type: application/octet-stream
Content-MD5: y8aU5niqcaWm2BXGqD9LdA==
Last-Modified: Wed, 21 May 2025 09:48:34 GMT
ETag: 0x8DD984CA760BBE1
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: ab1126d3-f01e-009b-6a35-ca1e6a000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Wed, 21 May 2025 09:49:30 GMT

+ check_backup_in_storage backup-azure-blob-sharded azure cfg
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ sed 's|azure://||'
++ sed 's|s3://||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O5YIhm6gFJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.441h3pJluZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O5YIhm6gFJ
++ cat /tmp/tmp.441h3pJluZ
++ rm /tmp/tmp.O5YIhm6gFJ /tmp/tmp.441h3pJluZ
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/cfg/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-05-21T09:49:31+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/cfg/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-physical-sharded/2025-05-21T09:48:13Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 19239    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 19239
Content-Type: application/octet-stream
Content-MD5: WWyutHuwDMMLjK08oNmOFw==
Last-Modified: Wed, 21 May 2025 09:48:30 GMT
ETag: 0x8DD984CA4CC66F6
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: 237c3e25-901e-00a2-0235-cae576000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Wed, 21 May 2025 09:49:31 GMT

+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-sharded to reach ready state.
+ '[' -z '' ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mOzho0rnhW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jLF1UIXeun
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mOzho0rnhW
++ cat /tmp/tmp.jLF1UIXeun
++ rm /tmp/tmp.mOzho0rnhW /tmp/tmp.jLF1UIXeun
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lPghtQYkJm
++ mktemp
+ local LAST_ERR=/tmp/tmp.OQnlDbhFW6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lPghtQYkJm
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("506279a9-507f-408e-ad80-ee252e6a0a41") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.OQnlDbhFW6
+ rm /tmp/tmp.lPghtQYkJm /tmp/tmp.OQnlDbhFW6
+ return 0
+ echo 'check backup and restore -- aws-s3'
check backup and restore -- aws-s3
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.z03HD55zbp
++ mktemp
+ local LAST_ERR=/tmp/tmp.LlTuzXtLIJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z03HD55zbp
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.LlTuzXtLIJ
+ rm /tmp/tmp.z03HD55zbp /tmp/tmp.LlTuzXtLIJ
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "requested" ....OK
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-8499", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.eoEI7MHZLT
++ mktemp
+ local LAST_ERR=/tmp/tmp.bk8OVYm24B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eoEI7MHZLT
+ cat /tmp/tmp.bk8OVYm24B
+ rm /tmp/tmp.eoEI7MHZLT /tmp/tmp.bk8OVYm24B
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "ready" ..OK
+ [[ 0 -eq 1 ]]
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mH7l2Izr4x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3RXUGQxbvU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mH7l2Izr4x
++ cat /tmp/tmp.3RXUGQxbvU
++ rm /tmp/tmp.mH7l2Izr4x /tmp/tmp.3RXUGQxbvU
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KvwtZZ2wBj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eKegYa378R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KvwtZZ2wBj
++ cat /tmp/tmp.eKegYa378R
++ rm /tmp/tmp.KvwtZZ2wBj /tmp/tmp.eKegYa378R
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sI3Qo0C7Ti
+++ mktemp
++ local LAST_ERR=/tmp/tmp.h8ICjVfYTr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sI3Qo0C7Ti
++ cat /tmp/tmp.h8ICjVfYTr
++ rm /tmp/tmp.sI3Qo0C7Ti /tmp/tmp.h8ICjVfYTr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.obJQW3w5wI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T5IfiF0UoP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.obJQW3w5wI
++ cat /tmp/tmp.T5IfiF0UoP
++ rm /tmp/tmp.obJQW3w5wI /tmp/tmp.T5IfiF0UoP
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l6cSAwUD6z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uG8PyUfrPs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l6cSAwUD6z
++ cat /tmp/tmp.uG8PyUfrPs
++ rm /tmp/tmp.l6cSAwUD6z /tmp/tmp.uG8PyUfrPs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YWBayx7WQS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LbDVn94dWb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YWBayx7WQS
++ cat /tmp/tmp.LbDVn94dWb
++ rm /tmp/tmp.YWBayx7WQS /tmp/tmp.LbDVn94dWb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DUmsA24HRi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QSPNFUPozv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DUmsA24HRi
++ cat /tmp/tmp.QSPNFUPozv
++ rm /tmp/tmp.DUmsA24HRi /tmp/tmp.QSPNFUPozv
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zTss8wcRi6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.haSHu76lJf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zTss8wcRi6
++ cat /tmp/tmp.haSHu76lJf
++ rm /tmp/tmp.zTss8wcRi6 /tmp/tmp.haSHu76lJf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Tk5l2uceM5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4mFbADD8Ue
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Tk5l2uceM5
++ cat /tmp/tmp.4mFbADD8Ue
++ rm /tmp/tmp.Tk5l2uceM5 /tmp/tmp.4mFbADD8Ue
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MOveIF5C3X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rqGpCmJlSB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MOveIF5C3X
++ cat /tmp/tmp.rqGpCmJlSB
++ rm /tmp/tmp.MOveIF5C3X /tmp/tmp.rqGpCmJlSB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QVkZvDhred
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9qbMbj7Rwg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QVkZvDhred
++ cat /tmp/tmp.9qbMbj7Rwg
++ rm /tmp/tmp.QVkZvDhred /tmp/tmp.9qbMbj7Rwg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SUuU1ClKKf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g2oGDwzdLZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SUuU1ClKKf
++ cat /tmp/tmp.g2oGDwzdLZ
++ rm /tmp/tmp.SUuU1ClKKf /tmp/tmp.g2oGDwzdLZ
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.................................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cR7vgNCmsy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R7D6p4dvGa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cR7vgNCmsy
++ cat /tmp/tmp.R7D6p4dvGa
++ rm /tmp/tmp.cR7vgNCmsy /tmp/tmp.R7D6p4dvGa
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PdsSNvGnDz
++ mktemp
+ local LAST_ERR=/tmp/tmp.8LTo0Qv871
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PdsSNvGnDz
+ cat /tmp/tmp.8LTo0Qv871
+ rm /tmp/tmp.PdsSNvGnDz /tmp/tmp.8LTo0Qv871
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.6i402Be8ap/find-sharded
+ check_exported_mongos_service_endpoint 35.232.179.101
+ local host=35.232.179.101
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YKmV915Ikq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3QIkEv3E8q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YKmV915Ikq
++ cat /tmp/tmp.3QIkEv3E8q
++ rm /tmp/tmp.YKmV915Ikq /tmp/tmp.3QIkEv3E8q
++ return 0
+ '[' 35.232.179.101 '!=' 35.232.179.101 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5YiiO0H4wT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rtBLnXRKlw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5YiiO0H4wT
++ cat /tmp/tmp.rtBLnXRKlw
++ rm /tmp/tmp.5YiiO0H4wT /tmp/tmp.rtBLnXRKlw
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.69pZkwSQGP
++ mktemp
+ local LAST_ERR=/tmp/tmp.YjgnhhPsW4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.69pZkwSQGP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f14e3a8d-3530-419e-9497-de92c02e30f6") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.YjgnhhPsW4
+ rm /tmp/tmp.69pZkwSQGP /tmp/tmp.YjgnhhPsW4
+ return 0
+ echo 'check backup and restore -- gcp-cs'
check backup and restore -- gcp-cs
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
+ local LAST_OUT=/tmp/tmp.YRJymv9cu7
++ mktemp
+ local LAST_ERR=/tmp/tmp.wIlXpFN9lW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YRJymv9cu7
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.wIlXpFN9lW
+ rm /tmp/tmp.YRJymv9cu7 /tmp/tmp.wIlXpFN9lW
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "requested" ....OK
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-8499", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RldK0NUcsd
++ mktemp
+ local LAST_ERR=/tmp/tmp.BzM9vgMYdV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RldK0NUcsd
+ cat /tmp/tmp.BzM9vgMYdV
+ rm /tmp/tmp.RldK0NUcsd /tmp/tmp.BzM9vgMYdV
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "ready" ..OK
+ [[ 0 -eq 1 ]]
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4bhcgE7g6G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sGVGxCRsVc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4bhcgE7g6G
++ cat /tmp/tmp.sGVGxCRsVc
++ rm /tmp/tmp.4bhcgE7g6G /tmp/tmp.sGVGxCRsVc
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vkWO9CrrRH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lesOvVr50d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vkWO9CrrRH
++ cat /tmp/tmp.lesOvVr50d
++ rm /tmp/tmp.vkWO9CrrRH /tmp/tmp.lesOvVr50d
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cposopVhCU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GBgIlHgHPT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cposopVhCU
++ cat /tmp/tmp.GBgIlHgHPT
++ rm /tmp/tmp.cposopVhCU /tmp/tmp.GBgIlHgHPT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qclzyCUM94
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d3n8v3uiTF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qclzyCUM94
++ cat /tmp/tmp.d3n8v3uiTF
++ rm /tmp/tmp.qclzyCUM94 /tmp/tmp.d3n8v3uiTF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Trll5fchYB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gXYAUmGWFF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Trll5fchYB
++ cat /tmp/tmp.gXYAUmGWFF
++ rm /tmp/tmp.Trll5fchYB /tmp/tmp.gXYAUmGWFF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0gq5DpP3PM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rCxBrLuOhb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0gq5DpP3PM
++ cat /tmp/tmp.rCxBrLuOhb
++ rm /tmp/tmp.0gq5DpP3PM /tmp/tmp.rCxBrLuOhb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DDHC3SNqLU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JlYibNp4W3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DDHC3SNqLU
++ cat /tmp/tmp.JlYibNp4W3
++ rm /tmp/tmp.DDHC3SNqLU /tmp/tmp.JlYibNp4W3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zc4kJ2zQCj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ajuFOyBBpQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zc4kJ2zQCj
++ cat /tmp/tmp.ajuFOyBBpQ
++ rm /tmp/tmp.zc4kJ2zQCj /tmp/tmp.ajuFOyBBpQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.khHUbRN8Kd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ta27GeBnOG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.khHUbRN8Kd
++ cat /tmp/tmp.Ta27GeBnOG
++ rm /tmp/tmp.khHUbRN8Kd /tmp/tmp.Ta27GeBnOG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qBhz8YmgkY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WcpeH6BeWA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qBhz8YmgkY
++ cat /tmp/tmp.WcpeH6BeWA
++ rm /tmp/tmp.qBhz8YmgkY /tmp/tmp.WcpeH6BeWA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OzkbcdNTcO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rSXLnPW63n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OzkbcdNTcO
++ cat /tmp/tmp.rSXLnPW63n
++ rm /tmp/tmp.OzkbcdNTcO /tmp/tmp.rSXLnPW63n
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2OtWtfu5ef
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LWgY6trfbl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2OtWtfu5ef
++ cat /tmp/tmp.LWgY6trfbl
++ rm /tmp/tmp.2OtWtfu5ef /tmp/tmp.LWgY6trfbl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K599XRxVKP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aEOW6A3GZt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K599XRxVKP
++ cat /tmp/tmp.aEOW6A3GZt
++ rm /tmp/tmp.K599XRxVKP /tmp/tmp.aEOW6A3GZt
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.................................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bb3kc4r3H0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UQNLWt4S5t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bb3kc4r3H0
++ cat /tmp/tmp.UQNLWt4S5t
++ rm /tmp/tmp.bb3kc4r3H0 /tmp/tmp.UQNLWt4S5t
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JyLGfwzOum
++ mktemp
+ local LAST_ERR=/tmp/tmp.rOpgfLPn9K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JyLGfwzOum
+ cat /tmp/tmp.rOpgfLPn9K
+ rm /tmp/tmp.JyLGfwzOum /tmp/tmp.rOpgfLPn9K
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.6i402Be8ap/find-sharded
+ check_exported_mongos_service_endpoint 35.232.179.101
+ local host=35.232.179.101
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L2pR3gV5Z2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cPYHWBcVWK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L2pR3gV5Z2
++ cat /tmp/tmp.cPYHWBcVWK
++ rm /tmp/tmp.L2pR3gV5Z2 /tmp/tmp.cPYHWBcVWK
++ return 0
+ '[' 35.232.179.101 '!=' 35.232.179.101 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D5hJXX9Uzb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HfRMvXom5y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D5hJXX9Uzb
++ cat /tmp/tmp.HfRMvXom5y
++ rm /tmp/tmp.D5hJXX9Uzb /tmp/tmp.HfRMvXom5y
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kbBtuOF8S2
++ mktemp
+ local LAST_ERR=/tmp/tmp.7kLW9ss16l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kbBtuOF8S2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2d59887b-63b3-4b54-858d-f1e158228106") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.7kLW9ss16l
+ rm /tmp/tmp.kbBtuOF8S2 /tmp/tmp.7kLW9ss16l
+ return 0
+ echo 'check backup and restore -- azure-blob'
check backup and restore -- azure-blob
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vyfAlQPqUZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.wzhJJe5izD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vyfAlQPqUZ
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.wzhJJe5izD
+ rm /tmp/tmp.vyfAlQPqUZ /tmp/tmp.wzhJJe5izD
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "requested" ....OK
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-8499", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5JOC2zflhO
++ mktemp
+ local LAST_ERR=/tmp/tmp.VdSi8gPR4U
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5JOC2zflhO
+ cat /tmp/tmp.VdSi8gPR4U
+ rm /tmp/tmp.5JOC2zflhO /tmp/tmp.VdSi8gPR4U
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.6i402Be8ap/statefulset_some-name-rs0.yml
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "ready" ..OK
+ [[ 0 -eq 1 ]]
++ yq '.metadata.annotations."percona.com/resync-pbm"'
++ kubectl_bin get psmdb some-name -o yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xwcsiJHbBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2rfxsACz0i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xwcsiJHbBI
++ cat /tmp/tmp.2rfxsACz0i
++ rm /tmp/tmp.xwcsiJHbBI /tmp/tmp.2rfxsACz0i
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VX573Y93h2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6GD8ogu4OX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VX573Y93h2
++ cat /tmp/tmp.6GD8ogu4OX
++ rm /tmp/tmp.VX573Y93h2 /tmp/tmp.6GD8ogu4OX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sb7xN1qCJb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.scel1FLdA4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sb7xN1qCJb
++ cat /tmp/tmp.scel1FLdA4
++ rm /tmp/tmp.sb7xN1qCJb /tmp/tmp.scel1FLdA4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bldIOHLQiQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mZAGXJSd0S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bldIOHLQiQ
++ cat /tmp/tmp.mZAGXJSd0S
++ rm /tmp/tmp.bldIOHLQiQ /tmp/tmp.mZAGXJSd0S
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5fTFlnax7K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fJ0nAzEiSz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5fTFlnax7K
++ cat /tmp/tmp.fJ0nAzEiSz
++ rm /tmp/tmp.5fTFlnax7K /tmp/tmp.fJ0nAzEiSz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rJ7Lk5k6Rq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LeujPlWcnk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rJ7Lk5k6Rq
++ cat /tmp/tmp.LeujPlWcnk
++ rm /tmp/tmp.rJ7Lk5k6Rq /tmp/tmp.LeujPlWcnk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rQ3PHKTgWR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OyK3o714lc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rQ3PHKTgWR
++ cat /tmp/tmp.OyK3o714lc
++ rm /tmp/tmp.rQ3PHKTgWR /tmp/tmp.OyK3o714lc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D5MRdOMxn1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U3xoDsP3LG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D5MRdOMxn1
++ cat /tmp/tmp.U3xoDsP3LG
++ rm /tmp/tmp.D5MRdOMxn1 /tmp/tmp.U3xoDsP3LG
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FyQuR3f2hE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lTJq3saxL8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FyQuR3f2hE
++ cat /tmp/tmp.lTJq3saxL8
++ rm /tmp/tmp.FyQuR3f2hE /tmp/tmp.lTJq3saxL8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BhwzZAZwUY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b72eDdJLAc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BhwzZAZwUY
++ cat /tmp/tmp.b72eDdJLAc
++ rm /tmp/tmp.BhwzZAZwUY /tmp/tmp.b72eDdJLAc
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local mongo_bin=mongo
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2OIFM6x8CU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IeF1up7LMH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2OIFM6x8CU
++ cat /tmp/tmp.IeF1up7LMH
++ rm /tmp/tmp.2OIFM6x8CU /tmp/tmp.IeF1up7LMH
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Fvhg77lsvM
++ mktemp
+ local LAST_ERR=/tmp/tmp.eMh3GHvrlO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Fvhg77lsvM
+ cat /tmp/tmp.eMh3GHvrlO
+ rm /tmp/tmp.Fvhg77lsvM /tmp/tmp.eMh3GHvrlO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.6i402Be8ap/find-sharded
+ check_exported_mongos_service_endpoint 35.232.179.101
+ local host=35.232.179.101
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ODeZGiZlnI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.36M3Y6bT22
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ODeZGiZlnI
++ cat /tmp/tmp.36M3Y6bT22
++ rm /tmp/tmp.ODeZGiZlnI /tmp/tmp.36M3Y6bT22
++ return 0
+ '[' 35.232.179.101 '!=' 35.232.179.101 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.53MDnFia4s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GHr1eJHgx7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.53MDnFia4s
++ cat /tmp/tmp.GHr1eJHgx7
++ rm /tmp/tmp.53MDnFia4s /tmp/tmp.GHr1eJHgx7
++ return 0
+ local client_container=psmdb-client-b9788d8bc-zdnnj
+ kubectl_bin exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GHyAkuCzLH
++ mktemp
+ local LAST_ERR=/tmp/tmp.M7DXKBaXWK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-zdnnj -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GHyAkuCzLH
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-8499.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("32008cea-0f94-4d7b-8be7-f8eed0a1a3ed") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.M7DXKBaXWK
+ rm /tmp/tmp.GHyAkuCzLH /tmp/tmp.M7DXKBaXWK
+ return 0
+ echo 'check backup and restore -- minio'
check backup and restore -- minio
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I1tWibM21m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j1XKZa04la
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I1tWibM21m
++ cat /tmp/tmp.j1XKZa04la
++ rm /tmp/tmp.I1tWibM21m /tmp/tmp.j1XKZa04la
++ return 0
+ backup_dest_minio=operator-testing/2025-05-21T09:48:40Z
+ run_restore backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-sharded/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1915/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.nr6euBpAoJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.3YFTIhpHVz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nr6euBpAoJ
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-sharded created
+ cat /tmp/tmp.3YFTIhpHVz
+ rm /tmp/tmp.nr6euBpAoJ /tmp/tmp.3YFTIhpHVz
+ return 0
+ run_recovery_check backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-minio-sharded some-name requested 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-minio-sharded to reach state "requested" ....