Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/logs/demand-backup-physical-sharded.log
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-physical-sharded-32047
+ local ns=demand-backup-physical-sharded-32047
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.7NDXnELFIU
++ mktemp
+ local LAST_ERR=/tmp/tmp.fUZqeyEMRd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7NDXnELFIU
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.fUZqeyEMRd
+ rm /tmp/tmp.7NDXnELFIU /tmp/tmp.fUZqeyEMRd
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27188 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27188 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.F7ldkTI457
++ mktemp
+ local LAST_ERR=/tmp/tmp.38lGf3J7tu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F7ldkTI457
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.38lGf3J7tu
+ rm /tmp/tmp.F7ldkTI457 /tmp/tmp.38lGf3J7tu
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.2caclDcLDZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.9568Gvp5Pv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2caclDcLDZ
+ cat /tmp/tmp.9568Gvp5Pv
+ rm /tmp/tmp.2caclDcLDZ /tmp/tmp.9568Gvp5Pv
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.fJtAJQafVG
++ mktemp
+ local LAST_ERR=/tmp/tmp.sNCstQD6OU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fJtAJQafVG
+ cat /tmp/tmp.sNCstQD6OU
+ rm /tmp/tmp.fJtAJQafVG /tmp/tmp.sNCstQD6OU
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ou78MFPmyk
++ mktemp
+ local LAST_ERR=/tmp/tmp.nC4Dq3Hqbo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ou78MFPmyk
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.nC4Dq3Hqbo
+ rm /tmp/tmp.ou78MFPmyk /tmp/tmp.nC4Dq3Hqbo
+ return 0
+ check_crd_for_deletion PR-1735-753d0dfe
+ local git_tag=PR-1735-753d0dfe
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1735-753d0dfe/deploy/crd.yaml
++ yq eval .metadata.name
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OBgfSEFAOc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fsPJBqjnCZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.OBgfSEFAOc
++ cat /tmp/tmp.fsPJBqjnCZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.OBgfSEFAOc
++ cat /tmp/tmp.fsPJBqjnCZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.OBgfSEFAOc
++ cat /tmp/tmp.fsPJBqjnCZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.OBgfSEFAOc
++ cat /tmp/tmp.fsPJBqjnCZ
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.OBgfSEFAOc /tmp/tmp.fsPJBqjnCZ
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ xargs kubectl delete ns
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.VwmdGJIrFK
++ mktemp
+ local LAST_ERR=/tmp/tmp.Cl29bDTRVN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.o8F9olAgpN
++ mktemp
+ local LAST_ERR=/tmp/tmp.LCOLzZgTO2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VwmdGJIrFK
+ cat /tmp/tmp.Cl29bDTRVN
+ rm /tmp/tmp.VwmdGJIrFK /tmp/tmp.Cl29bDTRVN
+ return 0
namespace "demand-backup-physical-sharded-27188" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.o8F9olAgpN
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.LCOLzZgTO2
+ rm /tmp/tmp.o8F9olAgpN /tmp/tmp.LCOLzZgTO2
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qAVH3efiYM
++ mktemp
+ local LAST_ERR=/tmp/tmp.rL3yAaRq38
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qAVH3efiYM
+ cat /tmp/tmp.rL3yAaRq38
+ rm /tmp/tmp.qAVH3efiYM /tmp/tmp.rL3yAaRq38
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.BRHFIodF6r
++ mktemp
+ local LAST_ERR=/tmp/tmp.dxj7HaXZL7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BRHFIodF6r
namespace/psmdb-operator created
+ cat /tmp/tmp.dxj7HaXZL7
+ rm /tmp/tmp.BRHFIodF6r /tmp/tmp.dxj7HaXZL7
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YYlVfoUu4b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FCPmRlihKx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YYlVfoUu4b
++ cat /tmp/tmp.FCPmRlihKx
++ rm /tmp/tmp.YYlVfoUu4b /tmp/tmp.FCPmRlihKx
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.heGE69ECmo
++ mktemp
+ local LAST_ERR=/tmp/tmp.bIdngtyPh5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.heGE69ECmo
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3" modified.
+ cat /tmp/tmp.bIdngtyPh5
+ rm /tmp/tmp.heGE69ECmo /tmp/tmp.bIdngtyPh5
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
++ mktemp
+ local temp_operator_yaml=/tmp/tmp.y4N7fzULzX
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.l5wwq6Ik5g
++ mktemp
+ local LAST_ERR=/tmp/tmp.YXS23GmJeZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l5wwq6Ik5g
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.YXS23GmJeZ
+ rm /tmp/tmp.l5wwq6Ik5g /tmp/tmp.YXS23GmJeZ
+ return 0
+ [[ -n psmdb-operator ]]
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.MlafaJf8g4
++ mktemp
+ local LAST_ERR=/tmp/tmp.tRvMhWK4CD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MlafaJf8g4
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.tRvMhWK4CD
+ rm /tmp/tmp.MlafaJf8g4 /tmp/tmp.tRvMhWK4CD
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1735-753d0dfe") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/deploy/cw-operator.yaml
+ [[ amd64 == \a\r\m\6\4 ]]
+ kubectl_bin apply -f /tmp/tmp.y4N7fzULzX
++ mktemp
+ local LAST_OUT=/tmp/tmp.tqzdBEoBXK
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZNc44M5vuc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /tmp/tmp.y4N7fzULzX
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tqzdBEoBXK
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.ZNc44M5vuc
+ rm /tmp/tmp.tqzdBEoBXK /tmp/tmp.ZNc44M5vuc
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T0HbVDZYuW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gm77bF8Fwl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T0HbVDZYuW
++ cat /tmp/tmp.Gm77bF8Fwl
++ rm /tmp/tmp.T0HbVDZYuW /tmp/tmp.Gm77bF8Fwl
++ return 0
+ wait_pod percona-server-mongodb-operator-8c87d9689-lffbw
+ local pod=percona-server-mongodb-operator-8c87d9689-lffbw
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-8c87d9689-lffbw to be ready..OK
+ create_namespace demand-backup-physical-sharded-32047
+ local namespace=demand-backup-physical-sharded-32047
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-physical-sharded-32047'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-sharded-32047
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-physical-sharded-32047 --ignore-not-found
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns
+ local LAST_OUT=/tmp/tmp.dPHo733vxo
+ kubectl_bin get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.Svy0VjHpXD
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-physical-sharded-32047 --ignore-not-found
+ local LAST_OUT=/tmp/tmp.Dmt4rEUDbn
++ mktemp
+ local LAST_ERR=/tmp/tmp.AezR49BWNE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dPHo733vxo
+ cat /tmp/tmp.Svy0VjHpXD
+ rm /tmp/tmp.dPHo733vxo /tmp/tmp.Svy0VjHpXD
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-32047
++ mktemp
+ local LAST_OUT=/tmp/tmp.69CriIyDJL
++ mktemp
+ local LAST_ERR=/tmp/tmp.vXCjkwqbq0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-sharded-32047
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Dmt4rEUDbn
+ cat /tmp/tmp.AezR49BWNE
+ rm /tmp/tmp.Dmt4rEUDbn /tmp/tmp.AezR49BWNE
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.69CriIyDJL
+ cat /tmp/tmp.vXCjkwqbq0
+ rm /tmp/tmp.69CriIyDJL /tmp/tmp.vXCjkwqbq0
+ return 0
+ desc 'create namespace demand-backup-physical-sharded-32047'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-physical-sharded-32047
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-physical-sharded-32047
++ mktemp
+ local LAST_OUT=/tmp/tmp.8yfw6BPykE
++ mktemp
+ local LAST_ERR=/tmp/tmp.iB382r6ppK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-physical-sharded-32047
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8yfw6BPykE
namespace/demand-backup-physical-sharded-32047 created
+ cat /tmp/tmp.iB382r6ppK
+ rm /tmp/tmp.8yfw6BPykE /tmp/tmp.iB382r6ppK
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.on4ALf3UFY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QE2LiuUuaL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
namespace "gmp-system" deleted
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.on4ALf3UFY
++ cat /tmp/tmp.QE2LiuUuaL
++ rm /tmp/tmp.on4ALf3UFY /tmp/tmp.QE2LiuUuaL
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3 --namespace=demand-backup-physical-sharded-32047
++ mktemp
+ local LAST_OUT=/tmp/tmp.SoAwYsw9Uq
++ mktemp
+ local LAST_ERR=/tmp/tmp.G5F2CXY477
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3 --namespace=demand-backup-physical-sharded-32047
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SoAwYsw9Uq
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1735-753d0dfe-52-cluster3" modified.
+ cat /tmp/tmp.G5F2CXY477
+ rm /tmp/tmp.SoAwYsw9Uq /tmp/tmp.G5F2CXY477
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false --set 'tolerations[0].key=kubernetes.io/arch' --set 'tolerations[0].operator=Equal' --set 'tolerations[0].value=arm64' --set 'tolerations[0].effect=NoSchedule' --set 'postJob.tolerations[0].key=kubernetes.io/arch' --set 'postJob.tolerations[0].operator=Equal' --set 'postJob.tolerations[0].value=arm64' --set 'postJob.tolerations[0].effect=NoSchedule' minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false --set 'tolerations[0].key=kubernetes.io/arch' --set 'tolerations[0].operator=Equal' --set 'tolerations[0].value=arm64' --set 'tolerations[0].effect=NoSchedule' --set 'postJob.tolerations[0].key=kubernetes.io/arch' --set 'postJob.tolerations[0].operator=Equal' --set 'postJob.tolerations[0].value=arm64' --set 'postJob.tolerations[0].effect=NoSchedule' minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Jan 13 14:32:14 2025
NAMESPACE: demand-backup-physical-sharded-32047
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-physical-sharded-32047.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-sharded-32047 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-sharded-32047

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-sharded-32047 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-sharded-32047 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aH3NPKY0uK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cytVzDxPcL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aH3NPKY0uK
++ cat /tmp/tmp.cytVzDxPcL
++ rm /tmp/tmp.aH3NPKY0uK /tmp/tmp.cytVzDxPcL
++ return 0
+ MINIO_POD=minio-service-754d5879cb-fn8w8
+ wait_pod minio-service-754d5879cb-fn8w8
+ local pod=minio-service-754d5879cb-fn8w8
+ set +o xtrace
waiting for pod/minio-service-754d5879cb-fn8w8 to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-32047.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.Qr6Olx4XJ6
++ mktemp
+ local LAST_ERR=/tmp/tmp.sY7SvVkwgT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-32047.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Qr6Olx4XJ6
service/minio-service created
+ cat /tmp/tmp.sY7SvVkwgT
+ rm /tmp/tmp.Qr6Olx4XJ6 /tmp/tmp.sY7SvVkwgT
+ return 0
+ aws_cli 's3 mb s3://operator-testing'
+ local 'cmd=s3 mb s3://operator-testing'
+ kubectl_bin run -i --rm aws-cli --image=amazon/aws-cli --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 '--overrides={"apiVersion": "v1","spec": {"tolerations": [{"key": "kubernetes.io/arch","operator": "Equal","value": "arm64","effect": "NoSchedule"}]}}' -- --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.n0QWL6rA0y
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Mip4prrmd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=amazon/aws-cli --restart=Never --env=AWS_ACCESS_KEY_ID=some-access-key --env=AWS_SECRET_ACCESS_KEY=some-secret-key --env=AWS_DEFAULT_REGION=us-east-1 '--overrides={"apiVersion": "v1","spec": {"tolerations": [{"key": "kubernetes.io/arch","operator": "Equal","value": "arm64","effect": "NoSchedule"}]}}' -- --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n0QWL6rA0y
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.9Mip4prrmd
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-physical-sharded-32047
+ rm /tmp/tmp.n0QWL6rA0y /tmp/tmp.9Mip4prrmd
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tdScgipKA7
++ mktemp
+ local LAST_ERR=/tmp/tmp.ITv2Y88IiR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tdScgipKA7
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.ITv2Y88IiR
+ rm /tmp/tmp.tdScgipKA7 /tmp/tmp.ITv2Y88IiR
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.eoNhqbcEJr
++ mktemp
+ local LAST_ERR=/tmp/tmp.W9suIKdIoz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eoNhqbcEJr
secret/some-users created
+ cat /tmp/tmp.W9suIKdIoz
+ rm /tmp/tmp.eoNhqbcEJr /tmp/tmp.W9suIKdIoz
+ return 0
+ apply_client /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/client_with_tls.yml
+ [[ amd64 == \a\r\m\6\4 ]]
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.N4VKW3qtSl
++ mktemp
+ local LAST_ERR=/tmp/tmp.XtILqMSS1u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N4VKW3qtSl
deployment.apps/psmdb-client created
+ cat /tmp/tmp.XtILqMSS1u
+ rm /tmp/tmp.N4VKW3qtSl /tmp/tmp.XtILqMSS1u
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ [[ -n '' ]]
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
++ mktemp
++ mktemp
+ local temp_cr=/tmp/tmp.oWw9QRbHns
+ yq eval '
		(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0" |
		(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest" |
		(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1735-753d0dfe" |
		(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup" |
		.spec.upgradeOptions.apply="Never"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ local LAST_OUT=/tmp/tmp.g7LTOEsLE7
++ mktemp
+ local LAST_ERR=/tmp/tmp.V3dWCX810L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ [[ amd64 == \a\r\m\6\4 ]]
+ cat /tmp/tmp.oWw9QRbHns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g7LTOEsLE7
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.V3dWCX810L
+ rm /tmp/tmp.g7LTOEsLE7 /tmp/tmp.V3dWCX810L
+ return 0
+ echo 'check if all pods started'
check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.imFjIXNXWj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cvVki8OxHH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.imFjIXNXWj
++ cat /tmp/tmp.cvVki8OxHH
++ rm /tmp/tmp.imFjIXNXWj /tmp/tmp.cvVki8OxHH
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KSOP0Rc2r5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NVEZ6Nz2mV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KSOP0Rc2r5
++ cat /tmp/tmp.NVEZ6Nz2mV
++ rm /tmp/tmp.KSOP0Rc2r5 /tmp/tmp.NVEZ6Nz2mV
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.........................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zhGcCVkRCU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TM0SmfIky1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zhGcCVkRCU
++ cat /tmp/tmp.TM0SmfIky1
++ rm /tmp/tmp.zhGcCVkRCU /tmp/tmp.TM0SmfIky1
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PXjxr7wfGN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XboiIfs5DU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PXjxr7wfGN
++ cat /tmp/tmp.XboiIfs5DU
++ rm /tmp/tmp.PXjxr7wfGN /tmp/tmp.XboiIfs5DU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UUaWY5ghKJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aJyLiHjQD5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UUaWY5ghKJ
++ cat /tmp/tmp.aJyLiHjQD5
++ rm /tmp/tmp.UUaWY5ghKJ /tmp/tmp.aJyLiHjQD5
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R2LW32T1cp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5YtY7AYXSI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R2LW32T1cp
++ cat /tmp/tmp.5YtY7AYXSI
++ rm /tmp/tmp.R2LW32T1cp /tmp/tmp.5YtY7AYXSI
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yOyzqtpMVQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AvZB7Fnkfs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yOyzqtpMVQ
++ cat /tmp/tmp.AvZB7Fnkfs
++ rm /tmp/tmp.yOyzqtpMVQ /tmp/tmp.AvZB7Fnkfs
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9yjsX2cFCi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ISCyMCw2zi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9yjsX2cFCi
++ cat /tmp/tmp.ISCyMCw2zi
++ rm /tmp/tmp.9yjsX2cFCi /tmp/tmp.ISCyMCw2zi
++ return 0
+ lbEndpoint=35.225.99.246
+ '[' -z 35.225.99.246 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yaBzlTEgnW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BADRRRJG65
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yaBzlTEgnW
++ cat /tmp/tmp.BADRRRJG65
++ rm /tmp/tmp.yaBzlTEgnW /tmp/tmp.BADRRRJG65
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EwkkY0L9nZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.hMrrBsAbM2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EwkkY0L9nZ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b2f54817-7d68-42fe-8d64-851339fc7706") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.hMrrBsAbM2
+ rm /tmp/tmp.EwkkY0L9nZ /tmp/tmp.hMrrBsAbM2
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NOIEj0HHoj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ik4PaZ9X2P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NOIEj0HHoj
++ cat /tmp/tmp.Ik4PaZ9X2P
++ rm /tmp/tmp.NOIEj0HHoj /tmp/tmp.Ik4PaZ9X2P
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.m1far8gWSg
++ mktemp
+ local LAST_ERR=/tmp/tmp.14QsJxm82L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m1far8gWSg
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e139404b-2e88-418c-a3c8-e0f7942c39a4") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.14QsJxm82L
+ rm /tmp/tmp.m1far8gWSg /tmp/tmp.14QsJxm82L
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XbQyvsdK8H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WmSEbxqLEA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XbQyvsdK8H
++ cat /tmp/tmp.WmSEbxqLEA
++ rm /tmp/tmp.XbQyvsdK8H /tmp/tmp.WmSEbxqLEA
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.acEln5Hd5R
++ mktemp
+ local LAST_ERR=/tmp/tmp.aVAnp9oFOg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.acEln5Hd5R
+ cat /tmp/tmp.aVAnp9oFOg
+ rm /tmp/tmp.acEln5Hd5R /tmp/tmp.aVAnp9oFOg
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.bnqjeK5fQs/find-sharded
+ echo 'waiting 60 seconds for stable timestamp in wiredtiger'
waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ echo 'running backups'
running backups
+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ /usr/bin/sed -e 's/name:/name: backup-minio-sharded/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: minio/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.czSg0y3nxO
++ mktemp
+ local LAST_ERR=/tmp/tmp.5VHUWENjeH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.czSg0y3nxO
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.5VHUWENjeH
+ rm /tmp/tmp.czSg0y3nxO /tmp/tmp.5VHUWENjeH
+ return 0
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/storageName:/storageName: aws-s3/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1OkPdtB87D
++ mktemp
+ local LAST_ERR=/tmp/tmp.iOYdLjuEPl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1OkPdtB87D
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.iOYdLjuEPl
+ rm /tmp/tmp.1OkPdtB87D /tmp/tmp.iOYdLjuEPl
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2WXpJxWbHV
++ mktemp
+ local LAST_ERR=/tmp/tmp.AmuZ0k1iVl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2WXpJxWbHV
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.AmuZ0k1iVl
+ rm /tmp/tmp.2WXpJxWbHV /tmp/tmp.AmuZ0k1iVl
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: azure-blob/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.fhmNJyF1NR
++ mktemp
+ local LAST_ERR=/tmp/tmp.3PBuNjgWPO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fhmNJyF1NR
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.3PBuNjgWPO
+ rm /tmp/tmp.fhmNJyF1NR /tmp/tmp.3PBuNjgWPO
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state........................................
+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sharded to reach ready state........................
+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob-sharded to reach ready state......................
+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-sharded to reach ready state.
+ '[' -z '' ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.i3DNVVl2Nk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dH9FvDTbNj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.i3DNVVl2Nk
++ cat /tmp/tmp.dH9FvDTbNj
++ rm /tmp/tmp.i3DNVVl2Nk /tmp/tmp.dH9FvDTbNj
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.kx1e2qCaKP
++ mktemp
+ local LAST_ERR=/tmp/tmp.lIlKqLibo0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kx1e2qCaKP
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("668f7dd2-b2c8-4083-88e1-39165e28a95c") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.lIlKqLibo0
+ rm /tmp/tmp.kx1e2qCaKP /tmp/tmp.lIlKqLibo0
+ return 0
+ echo 'check backup and restore -- aws-s3'
check backup and restore -- aws-s3
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bYSasxD84T
++ mktemp
+ local LAST_ERR=/tmp/tmp.GtTZUWWJzC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bYSasxD84T
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.GtTZUWWJzC
+ rm /tmp/tmp.bYSasxD84T /tmp/tmp.GtTZUWWJzC
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-aws-s3-sharded to reach requested state...............................................................................................................................................................................................................................................................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-32047", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SHrnhB5uyT
++ mktemp
+ local LAST_ERR=/tmp/tmp.M5ghiN8Lt3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SHrnhB5uyT
+ cat /tmp/tmp.M5ghiN8Lt3
+ rm /tmp/tmp.SHrnhB5uyT /tmp/tmp.M5ghiN8Lt3
+ return 0
+ [[ amd64 == \a\r\m\6\4 ]]
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-aws-s3-sharded to reach ready state................................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.NJReAeNYIR
++ mktemp
+ local LAST_ERR=/tmp/tmp.1mKiIvfh0m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NJReAeNYIR
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-32047"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-01-13T14:32:54Z"
  generation: 2
  name: some-name
  namespace: demand-backup-physical-sharded-32047
  resourceVersion: "82911"
  uid: 8813238a-a091-4ae0-a7cd-19a1cd690776
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-physical-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      minio:
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.19.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  conditions:
  - lastTransitionTime: "2025-01-13T14:32:54Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-01-13T14:32:56Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-01-13T14:34:11Z"
    message: 'cfg: ready'
    reason: RSReady
    status: "True"
    type: ready
  host: 35.225.99.246
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.15-9
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.1mKiIvfh0m
+ rm /tmp/tmp.NJReAeNYIR /tmp/tmp.1mKiIvfh0m
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rkFJW9mJXZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zoQPTGCWfz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rkFJW9mJXZ
++ cat /tmp/tmp.zoQPTGCWfz
++ rm /tmp/tmp.rkFJW9mJXZ /tmp/tmp.zoQPTGCWfz
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B0LXpoPYiv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IapR5tbHnT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B0LXpoPYiv
++ cat /tmp/tmp.IapR5tbHnT
++ rm /tmp/tmp.B0LXpoPYiv /tmp/tmp.IapR5tbHnT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WCrqcduQXx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e4VRBmZSww
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WCrqcduQXx
++ cat /tmp/tmp.e4VRBmZSww
++ rm /tmp/tmp.WCrqcduQXx /tmp/tmp.e4VRBmZSww
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7HDG79Tzer
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yrTd1F0zeM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7HDG79Tzer
++ cat /tmp/tmp.yrTd1F0zeM
++ rm /tmp/tmp.7HDG79Tzer /tmp/tmp.yrTd1F0zeM
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YawbHk8nwd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZnFA6GWAL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YawbHk8nwd
++ cat /tmp/tmp.PZnFA6GWAL
++ rm /tmp/tmp.YawbHk8nwd /tmp/tmp.PZnFA6GWAL
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QH6hZwgdcU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AAlQ21Fupa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QH6hZwgdcU
++ cat /tmp/tmp.AAlQ21Fupa
++ rm /tmp/tmp.QH6hZwgdcU /tmp/tmp.AAlQ21Fupa
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k2VYgxqCkD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7j6MfnfqQT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.k2VYgxqCkD
++ cat /tmp/tmp.7j6MfnfqQT
++ rm /tmp/tmp.k2VYgxqCkD /tmp/tmp.7j6MfnfqQT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pg9NLdSuvn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BLE65DeS7s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pg9NLdSuvn
++ cat /tmp/tmp.BLE65DeS7s
++ rm /tmp/tmp.pg9NLdSuvn /tmp/tmp.BLE65DeS7s
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.599czpXunI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aMJo1iJyks
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.599czpXunI
++ cat /tmp/tmp.aMJo1iJyks
++ rm /tmp/tmp.599czpXunI /tmp/tmp.aMJo1iJyks
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zv7Ata46pe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.puqrDaySEZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zv7Ata46pe
++ cat /tmp/tmp.puqrDaySEZ
++ rm /tmp/tmp.zv7Ata46pe /tmp/tmp.puqrDaySEZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tslMJTSxhQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rRVNdE9hCy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tslMJTSxhQ
++ cat /tmp/tmp.rRVNdE9hCy
++ rm /tmp/tmp.tslMJTSxhQ /tmp/tmp.rRVNdE9hCy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AZ301DmGWb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.skEhNJEqRU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AZ301DmGWb
++ cat /tmp/tmp.skEhNJEqRU
++ rm /tmp/tmp.AZ301DmGWb /tmp/tmp.skEhNJEqRU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FVTOhw66CG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2L8M9m6msE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FVTOhw66CG
++ cat /tmp/tmp.2L8M9m6msE
++ rm /tmp/tmp.FVTOhw66CG /tmp/tmp.2L8M9m6msE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FeTHKqCz3k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yxKWQXH10g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FeTHKqCz3k
++ cat /tmp/tmp.yxKWQXH10g
++ rm /tmp/tmp.FeTHKqCz3k /tmp/tmp.yxKWQXH10g
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jVCdDIBCuv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pisSkPyY55
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jVCdDIBCuv
++ cat /tmp/tmp.pisSkPyY55
++ rm /tmp/tmp.jVCdDIBCuv /tmp/tmp.pisSkPyY55
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c7KXyTypaf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U0SkbYJotE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c7KXyTypaf
++ cat /tmp/tmp.U0SkbYJotE
++ rm /tmp/tmp.c7KXyTypaf /tmp/tmp.U0SkbYJotE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2MIFIq7fuV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ufT08NXQOz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2MIFIq7fuV
++ cat /tmp/tmp.ufT08NXQOz
++ rm /tmp/tmp.2MIFIq7fuV /tmp/tmp.ufT08NXQOz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.m15Q5u49KG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9gWMRB4hRI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.m15Q5u49KG
++ cat /tmp/tmp.9gWMRB4hRI
++ rm /tmp/tmp.m15Q5u49KG /tmp/tmp.9gWMRB4hRI
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.............................................................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rbCGRt4yPp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OFjshjmrRE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rbCGRt4yPp
++ cat /tmp/tmp.OFjshjmrRE
++ rm /tmp/tmp.rbCGRt4yPp /tmp/tmp.OFjshjmrRE
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.T4pkgVIAY1
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ig7QIz3xJ8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T4pkgVIAY1
+ cat /tmp/tmp.Ig7QIz3xJ8
+ rm /tmp/tmp.T4pkgVIAY1 /tmp/tmp.Ig7QIz3xJ8
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.bnqjeK5fQs/find-sharded
+ check_exported_mongos_service_endpoint 35.225.99.246
+ local host=35.225.99.246
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sTUazEvbtq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BojW4tI4wd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sTUazEvbtq
++ cat /tmp/tmp.BojW4tI4wd
++ rm /tmp/tmp.sTUazEvbtq /tmp/tmp.BojW4tI4wd
++ return 0
+ '[' 35.225.99.246 '!=' 35.225.99.246 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9npYlJdy6C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YeoLVZqxCe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9npYlJdy6C
++ cat /tmp/tmp.YeoLVZqxCe
++ rm /tmp/tmp.9npYlJdy6C /tmp/tmp.YeoLVZqxCe
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pswvHDnqyW
++ mktemp
+ local LAST_ERR=/tmp/tmp.xWvTN2osac
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pswvHDnqyW
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("87b917df-4cff-4f55-a758-de1e7b59a42e") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.xWvTN2osac
+ rm /tmp/tmp.pswvHDnqyW /tmp/tmp.xWvTN2osac
+ return 0
+ echo 'check backup and restore -- gcp-cs'
check backup and restore -- gcp-cs
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
+ /usr/bin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XJtVdJ13dy
++ mktemp
+ local LAST_ERR=/tmp/tmp.qLpSEqQ3t0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XJtVdJ13dy
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.qLpSEqQ3t0
+ rm /tmp/tmp.XJtVdJ13dy /tmp/tmp.qLpSEqQ3t0
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach requested state...........................................................................................................................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-32047", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.42aNRnhg0k
++ mktemp
+ local LAST_ERR=/tmp/tmp.MyaTz1f09F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.42aNRnhg0k
+ cat /tmp/tmp.MyaTz1f09F
+ rm /tmp/tmp.42aNRnhg0k /tmp/tmp.MyaTz1f09F
+ return 0
+ [[ amd64 == \a\r\m\6\4 ]]
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach ready state......................................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yeU9w9Zz4E
++ mktemp
+ local LAST_ERR=/tmp/tmp.8DPfT3c35d
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yeU9w9Zz4E
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-32047"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-01-13T14:32:54Z"
  generation: 2
  name: some-name
  namespace: demand-backup-physical-sharded-32047
  resourceVersion: "95806"
  uid: 8813238a-a091-4ae0-a7cd-19a1cd690776
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-physical-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      minio:
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.19.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  conditions:
  - lastTransitionTime: "2025-01-13T14:32:54Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-01-13T14:32:56Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-01-13T14:34:11Z"
    message: 'cfg: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-01-13T14:56:21Z"
    message: |-
      dial: ping mongo: server selection error: context deadline exceeded, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, { Addr: some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-1.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, { Addr: some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-2.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, ] }
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, ] }
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 35.225.99.246
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.15-9
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.8DPfT3c35d
+ rm /tmp/tmp.yeU9w9Zz4E /tmp/tmp.8DPfT3c35d
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U6E6y6YGsq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z55lGo8VqM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U6E6y6YGsq
++ cat /tmp/tmp.Z55lGo8VqM
++ rm /tmp/tmp.U6E6y6YGsq /tmp/tmp.Z55lGo8VqM
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WsOXbNmpf2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nEsO0uVuU8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WsOXbNmpf2
++ cat /tmp/tmp.nEsO0uVuU8
++ rm /tmp/tmp.WsOXbNmpf2 /tmp/tmp.nEsO0uVuU8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.T5zC3lPlxN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vB8qxJbyeF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.T5zC3lPlxN
++ cat /tmp/tmp.vB8qxJbyeF
++ rm /tmp/tmp.T5zC3lPlxN /tmp/tmp.vB8qxJbyeF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CZ753ZggNV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hKLI5cRyFX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CZ753ZggNV
++ cat /tmp/tmp.hKLI5cRyFX
++ rm /tmp/tmp.CZ753ZggNV /tmp/tmp.hKLI5cRyFX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JkVb6l4QQC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XVw66DSdNK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JkVb6l4QQC
++ cat /tmp/tmp.XVw66DSdNK
++ rm /tmp/tmp.JkVb6l4QQC /tmp/tmp.XVw66DSdNK
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.r5VNmJRG1r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AmkP2qMnOB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.r5VNmJRG1r
++ cat /tmp/tmp.AmkP2qMnOB
++ rm /tmp/tmp.r5VNmJRG1r /tmp/tmp.AmkP2qMnOB
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hpylDGR77j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EmF3IezSzL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hpylDGR77j
++ cat /tmp/tmp.EmF3IezSzL
++ rm /tmp/tmp.hpylDGR77j /tmp/tmp.EmF3IezSzL
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qRcfFTkyaM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zePIxUsSfS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qRcfFTkyaM
++ cat /tmp/tmp.zePIxUsSfS
++ rm /tmp/tmp.qRcfFTkyaM /tmp/tmp.zePIxUsSfS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.klFQUeAfQE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LMS9p376Wk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.klFQUeAfQE
++ cat /tmp/tmp.LMS9p376Wk
++ rm /tmp/tmp.klFQUeAfQE /tmp/tmp.LMS9p376Wk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8y6AzttsnZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hIOJXsp286
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8y6AzttsnZ
++ cat /tmp/tmp.hIOJXsp286
++ rm /tmp/tmp.8y6AzttsnZ /tmp/tmp.hIOJXsp286
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.StjIcQBm5w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0loK1dfdHC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.StjIcQBm5w
++ cat /tmp/tmp.0loK1dfdHC
++ rm /tmp/tmp.StjIcQBm5w /tmp/tmp.0loK1dfdHC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUBp442F1F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AsKWFXg1v4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EUBp442F1F
++ cat /tmp/tmp.AsKWFXg1v4
++ rm /tmp/tmp.EUBp442F1F /tmp/tmp.AsKWFXg1v4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUO9BHM5pt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NL49cdkTVB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EUO9BHM5pt
++ cat /tmp/tmp.NL49cdkTVB
++ rm /tmp/tmp.EUO9BHM5pt /tmp/tmp.NL49cdkTVB
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O07EAThcq4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4byjkozI37
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O07EAThcq4
++ cat /tmp/tmp.4byjkozI37
++ rm /tmp/tmp.O07EAThcq4 /tmp/tmp.4byjkozI37
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KSD1PHLjTV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qm6GxmAcj2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KSD1PHLjTV
++ cat /tmp/tmp.qm6GxmAcj2
++ rm /tmp/tmp.KSD1PHLjTV /tmp/tmp.qm6GxmAcj2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c1U6LWKORC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CjD5tQK83i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c1U6LWKORC
++ cat /tmp/tmp.CjD5tQK83i
++ rm /tmp/tmp.c1U6LWKORC /tmp/tmp.CjD5tQK83i
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MrUX9boUGe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f172IIhOVm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MrUX9boUGe
++ cat /tmp/tmp.f172IIhOVm
++ rm /tmp/tmp.MrUX9boUGe /tmp/tmp.f172IIhOVm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t8rfkSymle
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PsFDK0sQf6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t8rfkSymle
++ cat /tmp/tmp.PsFDK0sQf6
++ rm /tmp/tmp.t8rfkSymle /tmp/tmp.PsFDK0sQf6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 17 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9sPn7v7KaA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SgBJXfTwr7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9sPn7v7KaA
++ cat /tmp/tmp.SgBJXfTwr7
++ rm /tmp/tmp.9sPn7v7KaA /tmp/tmp.SgBJXfTwr7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish..........................................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Lg4ehvQrGo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hRuYuCmAGV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Lg4ehvQrGo
++ cat /tmp/tmp.hRuYuCmAGV
++ rm /tmp/tmp.Lg4ehvQrGo /tmp/tmp.hRuYuCmAGV
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dh3LzmJ2zs
++ mktemp
+ local LAST_ERR=/tmp/tmp.OkxiMHhQem
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dh3LzmJ2zs
+ cat /tmp/tmp.OkxiMHhQem
+ rm /tmp/tmp.dh3LzmJ2zs /tmp/tmp.OkxiMHhQem
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.bnqjeK5fQs/find-sharded
+ check_exported_mongos_service_endpoint 35.225.99.246
+ local host=35.225.99.246
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gx6p0LASgu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9QTS2PHxAj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Gx6p0LASgu
++ cat /tmp/tmp.9QTS2PHxAj
++ rm /tmp/tmp.Gx6p0LASgu /tmp/tmp.9QTS2PHxAj
++ return 0
+ '[' 35.225.99.246 '!=' 35.225.99.246 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.brpxRNEMQm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HQk0HcsawS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.brpxRNEMQm
++ cat /tmp/tmp.HQk0HcsawS
++ rm /tmp/tmp.brpxRNEMQm /tmp/tmp.HQk0HcsawS
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PpCjNssost
++ mktemp
+ local LAST_ERR=/tmp/tmp.pQILLyHjaV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PpCjNssost
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f96aaa8d-3a46-4c68-b21b-4f5bd444fa83") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.pQILLyHjaV
+ rm /tmp/tmp.PpCjNssost /tmp/tmp.pQILLyHjaV
+ return 0
+ echo 'check backup and restore -- azure-blob'
check backup and restore -- azure-blob
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.qXmMVWGTAI
++ mktemp
+ local LAST_ERR=/tmp/tmp.aEe4PTPVV9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qXmMVWGTAI
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.aEe4PTPVV9
+ rm /tmp/tmp.qXmMVWGTAI /tmp/tmp.aEe4PTPVV9
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-azure-blob-sharded to reach requested state..............................................................................................................................................................................................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-32047", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4V3d2EbIFj
++ mktemp
+ local LAST_ERR=/tmp/tmp.l7eX2Ktnv6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4V3d2EbIFj
+ cat /tmp/tmp.l7eX2Ktnv6
+ rm /tmp/tmp.4V3d2EbIFj /tmp/tmp.l7eX2Ktnv6
+ return 0
+ [[ amd64 == \a\r\m\6\4 ]]
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.bnqjeK5fQs/statefulset_some-name-rs0.yml
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-azure-blob-sharded to reach ready state.......................................................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BsIt3rx8aR
++ mktemp
+ local LAST_ERR=/tmp/tmp.5Mojx5OyR1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BsIt3rx8aR
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-32047"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mongodb-operator:main-backup","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n  mode: slowOp\n  slowOpThresholdMs: 100\nsecurity:\n  enableEncryption: true\n  redactClientLogData: false\nsetParameter:\n  ttlMonitorSleepSecs: 60\n  wiredTigerConcurrentReadTransactions: 128\n  wiredTigerConcurrentWriteTransactions: 128\nstorage:\n  engine: wiredTiger\n  wiredTiger:\n    collectionConfig:\n      blockCompressor: snappy\n    engineConfig:\n      directoryForIndexes: false\n      journalCompressor: snappy\n    indexConfig:\n      prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
    percona.com/resync-pbm: "true"
  creationTimestamp: "2025-01-13T14:32:54Z"
  generation: 2
  name: some-name
  namespace: demand-backup-physical-sharded-32047
  resourceVersion: "106907"
  uid: 8813238a-a091-4ae0-a7cd-19a1cd690776
spec:
  backup:
    enabled: true
    image: perconalab/percona-server-mongodb-operator:main-backup
    storages:
      aws-s3:
        s3:
          bucket: operator-testing
          credentialsSecret: aws-s3-secret
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      azure-blob:
        azure:
          container: operator-testing
          credentialsSecret: azure-secret
          prefix: psmdb-demand-backup-physical-sharded
        type: azure
      gcp-cs:
        s3:
          bucket: operator-testing
          credentialsSecret: gcp-cs-secret
          endpointUrl: https://storage.googleapis.com
          insecureSkipTLSVerify: false
          prefix: psmdb-demand-backup-physical-sharded
          region: us-east-1
        type: s3
      minio:
        s3:
          bucket: operator-testing
          credentialsSecret: minio-secret
          endpointUrl: http://minio-service:9000/
          insecureSkipTLSVerify: false
          region: us-east-1
        type: s3
    tasks:
    - compressionType: gzip
      enabled: true
      name: weekly
      schedule: 0 0 * * 0
      storageName: aws-s3
  crVersion: 1.19.0
  image: perconalab/percona-server-mongodb-operator:main-mongod7.0
  imagePullPolicy: Always
  replsets:
  - affinity:
      antiAffinityTopologyKey: none
    configuration: |
      operationProfiling:
        mode: slowOp
        slowOpThresholdMs: 100
      security:
        enableEncryption: true
        redactClientLogData: false
      setParameter:
        ttlMonitorSleepSecs: 60
        wiredTigerConcurrentReadTransactions: 128
        wiredTigerConcurrentWriteTransactions: 128
      storage:
        engine: wiredTiger
        wiredTiger:
          collectionConfig:
            blockCompressor: snappy
          engineConfig:
            directoryForIndexes: false
            journalCompressor: snappy
          indexConfig:
            prefixCompression: true
    expose:
      enabled: false
      type: ClusterIP
    name: rs0
    resources:
      limits:
        cpu: 500m
        memory: 1G
      requests:
        cpu: 100m
        memory: 0.1G
    size: 3
    volumeSpec:
      persistentVolumeClaim:
        resources:
          requests:
            storage: 3Gi
  secrets:
    users: some-users
  sharding:
    configsvrReplSet:
      affinity:
        antiAffinityTopologyKey: none
      configuration: |
        operationProfiling:
          mode: slowOp
          slowOpThresholdMs: 100
        security:
          enableEncryption: true
          redactClientLogData: false
        setParameter:
          ttlMonitorSleepSecs: 60
          wiredTigerConcurrentReadTransactions: 128
          wiredTigerConcurrentWriteTransactions: 128
        storage:
          engine: wiredTiger
          wiredTiger:
            collectionConfig:
              blockCompressor: snappy
            engineConfig:
              directoryForIndexes: false
              journalCompressor: snappy
            indexConfig:
              prefixCompression: true
      expose:
        enabled: false
        type: ClusterIP
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
      volumeSpec:
        persistentVolumeClaim:
          resources:
            requests:
              storage: 3Gi
    enabled: true
    mongos:
      affinity:
        antiAffinityTopologyKey: none
      expose:
        type: LoadBalancer
      resources:
        limits:
          cpu: 500m
          memory: 1G
        requests:
          cpu: 100m
          memory: 0.1G
      size: 3
  upgradeOptions:
    apply: Never
status:
  conditions:
  - lastTransitionTime: "2025-01-13T14:32:54Z"
    status: "True"
    type: sharding
  - lastTransitionTime: "2025-01-13T14:32:56Z"
    status: "True"
    type: initializing
  - lastTransitionTime: "2025-01-13T14:34:11Z"
    message: 'cfg: ready'
    reason: RSReady
    status: "True"
    type: ready
  - lastTransitionTime: "2025-01-13T14:56:21Z"
    message: |-
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: context deadline exceeded, current topology: { Type: Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, ] }
      handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-32047.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-32047.svc.cluster.local on 10.7.160.10:53: no such host }, ] }
    reason: ErrorReconcile
    status: "True"
    type: error
  host: 35.225.99.246
  mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
  mongoVersion: 7.0.15-9
  mongos:
    ready: 0
    size: 0
    status: initializing
  observedGeneration: 2
  ready: 6
  replsets:
    cfg:
      initialized: true
      ready: 3
      size: 3
      status: ready
    rs0:
      added_as_shard: true
      initialized: true
      ready: 3
      size: 3
      status: ready
  size: 6
  state: initializing
+ cat /tmp/tmp.5Mojx5OyR1
+ rm /tmp/tmp.BsIt3rx8aR /tmp/tmp.5Mojx5OyR1
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.W8hUoisRLt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A4E0JmaXoG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.W8hUoisRLt
++ cat /tmp/tmp.A4E0JmaXoG
++ rm /tmp/tmp.W8hUoisRLt /tmp/tmp.A4E0JmaXoG
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q1y4gMzZGv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tf2ay7JeBr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q1y4gMzZGv
++ cat /tmp/tmp.Tf2ay7JeBr
++ rm /tmp/tmp.q1y4gMzZGv /tmp/tmp.Tf2ay7JeBr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DbgCrDtN7D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MxNeHvL3fO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DbgCrDtN7D
++ cat /tmp/tmp.MxNeHvL3fO
++ rm /tmp/tmp.DbgCrDtN7D /tmp/tmp.MxNeHvL3fO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HonMKqklWH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zvBnWcvIz7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HonMKqklWH
++ cat /tmp/tmp.zvBnWcvIz7
++ rm /tmp/tmp.HonMKqklWH /tmp/tmp.zvBnWcvIz7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qujtgSgNvl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HmSyjVSly2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qujtgSgNvl
++ cat /tmp/tmp.HmSyjVSly2
++ rm /tmp/tmp.qujtgSgNvl /tmp/tmp.HmSyjVSly2
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6mXza8XaBn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ONOqQzM7Fc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6mXza8XaBn
++ cat /tmp/tmp.ONOqQzM7Fc
++ rm /tmp/tmp.6mXza8XaBn /tmp/tmp.ONOqQzM7Fc
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9lZiRghKJl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q1yOrDSW9w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9lZiRghKJl
++ cat /tmp/tmp.q1yOrDSW9w
++ rm /tmp/tmp.9lZiRghKJl /tmp/tmp.q1yOrDSW9w
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.evyte45aps
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TZaGjawuIj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.evyte45aps
++ cat /tmp/tmp.TZaGjawuIj
++ rm /tmp/tmp.evyte45aps /tmp/tmp.TZaGjawuIj
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8BQTJw9SGO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5S74S05knH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8BQTJw9SGO
++ cat /tmp/tmp.5S74S05knH
++ rm /tmp/tmp.8BQTJw9SGO /tmp/tmp.5S74S05knH
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9OFDrMgQqu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qmoog9eYVm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9OFDrMgQqu
++ cat /tmp/tmp.qmoog9eYVm
++ rm /tmp/tmp.9OFDrMgQqu /tmp/tmp.qmoog9eYVm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VK6dPMCAjD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ePHUYbF2Ly
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VK6dPMCAjD
++ cat /tmp/tmp.ePHUYbF2Ly
++ rm /tmp/tmp.VK6dPMCAjD /tmp/tmp.ePHUYbF2Ly
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qrLn3Mg8kf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JGyidvmuMD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qrLn3Mg8kf
++ cat /tmp/tmp.JGyidvmuMD
++ rm /tmp/tmp.qrLn3Mg8kf /tmp/tmp.JGyidvmuMD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9HlifKFst1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jIdrLrWecj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9HlifKFst1
++ cat /tmp/tmp.jIdrLrWecj
++ rm /tmp/tmp.9HlifKFst1 /tmp/tmp.jIdrLrWecj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PszLzRbg1H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wCLQJNi0Gu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PszLzRbg1H
++ cat /tmp/tmp.wCLQJNi0Gu
++ rm /tmp/tmp.PszLzRbg1H /tmp/tmp.wCLQJNi0Gu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Okco6yBWyl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sK7QdgOvKn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Okco6yBWyl
++ cat /tmp/tmp.sK7QdgOvKn
++ rm /tmp/tmp.Okco6yBWyl /tmp/tmp.sK7QdgOvKn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c0iXQoZ9gd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.arLVmMFewi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c0iXQoZ9gd
++ cat /tmp/tmp.arLVmMFewi
++ rm /tmp/tmp.c0iXQoZ9gd /tmp/tmp.arLVmMFewi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yqdHZbFILQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uT3m5j0TOJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yqdHZbFILQ
++ cat /tmp/tmp.uT3m5j0TOJ
++ rm /tmp/tmp.yqdHZbFILQ /tmp/tmp.uT3m5j0TOJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JwEZ8e3sI8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gKzXW6SztN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JwEZ8e3sI8
++ cat /tmp/tmp.gKzXW6SztN
++ rm /tmp/tmp.JwEZ8e3sI8 /tmp/tmp.gKzXW6SztN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 17 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x0cd4mXpAw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nzALOFqkYp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x0cd4mXpAw
++ cat /tmp/tmp.nzALOFqkYp
++ rm /tmp/tmp.x0cd4mXpAw /tmp/tmp.nzALOFqkYp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 18 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.igyG6D7OcM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iT1yEffOkO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.igyG6D7OcM
++ cat /tmp/tmp.iT1yEffOkO
++ rm /tmp/tmp.igyG6D7OcM /tmp/tmp.iT1yEffOkO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 19 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TMRQJL8zMc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jMFxHqAK55
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TMRQJL8zMc
++ cat /tmp/tmp.jMFxHqAK55
++ rm /tmp/tmp.TMRQJL8zMc /tmp/tmp.jMFxHqAK55
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish...........................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DexKUXHm4K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wYvn641S3q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DexKUXHm4K
++ cat /tmp/tmp.wYvn641S3q
++ rm /tmp/tmp.DexKUXHm4K /tmp/tmp.wYvn641S3q
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SAxMg4buYB
++ mktemp
+ local LAST_ERR=/tmp/tmp.dy1Gsjgxqs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SAxMg4buYB
+ cat /tmp/tmp.dy1Gsjgxqs
+ rm /tmp/tmp.SAxMg4buYB /tmp/tmp.dy1Gsjgxqs
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.bnqjeK5fQs/find-sharded
+ check_exported_mongos_service_endpoint 35.225.99.246
+ local host=35.225.99.246
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PrnzMG9S2x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q2nlPPclBo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PrnzMG9S2x
++ cat /tmp/tmp.q2nlPPclBo
++ rm /tmp/tmp.PrnzMG9S2x /tmp/tmp.q2nlPPclBo
++ return 0
+ '[' 35.225.99.246 '!=' 35.225.99.246 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ibJ9VkswR5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BTDfAAQso3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ibJ9VkswR5
++ cat /tmp/tmp.BTDfAAQso3
++ rm /tmp/tmp.ibJ9VkswR5 /tmp/tmp.BTDfAAQso3
++ return 0
+ local client_container=psmdb-client-5d456f66db-qqfqg
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SHa8R4tTH2
++ mktemp
+ local LAST_ERR=/tmp/tmp.iD4hoQJe4z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-5d456f66db-qqfqg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SHa8R4tTH2
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-32047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e927c6f4-1278-4366-b6dd-579c330a7e68") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.iD4hoQJe4z
+ rm /tmp/tmp.SHa8R4tTH2 /tmp/tmp.iD4hoQJe4z
+ return 0
+ echo 'check backup and restore -- minio'
check backup and restore -- minio
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.4SJTJxzWi0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ae9Hq5Xe7n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4SJTJxzWi0
++ cat /tmp/tmp.ae9Hq5Xe7n
++ rm /tmp/tmp.4SJTJxzWi0 /tmp/tmp.ae9Hq5Xe7n
++ return 0
+ backup_dest_minio=operator-testing/2025-01-13T14:37:50Z
+ run_restore backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-sharded/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1735/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ofRt3yndMN
++ mktemp
+ local LAST_ERR=/tmp/tmp.0dE7MoHH3z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ofRt3yndMN
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-sharded created
+ cat /tmp/tmp.0dE7MoHH3z
+ rm /tmp/tmp.ofRt3yndMN /tmp/tmp.0dE7MoHH3z
+ return 0
+ run_recovery_check backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-minio-sharded some-name requested 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/restore-backup-minio-sharded to reach requested state......................................................