Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/logs/replset-overrides.log
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ main
+ create_infra replset-overrides-23335
+ local ns=replset-overrides-23335
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.SDVoJXi8H6
++ mktemp
+ local LAST_ERR=/tmp/tmp.RgH6SHxpPo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SDVoJXi8H6
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.RgH6SHxpPo
+ rm /tmp/tmp.SDVoJXi8H6 /tmp/tmp.RgH6SHxpPo
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n replset-overrides-28539 backup-minio-physical --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-physical patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.n7EyuwiUeG
++ mktemp
+ local LAST_ERR=/tmp/tmp.RosA8YcC2V
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n7EyuwiUeG
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.RosA8YcC2V
+ rm /tmp/tmp.n7EyuwiUeG /tmp/tmp.RosA8YcC2V
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.kXOfgqSPM3
++ mktemp
+ local LAST_ERR=/tmp/tmp.sbKinxz6R8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kXOfgqSPM3
+ cat /tmp/tmp.sbKinxz6R8
+ rm /tmp/tmp.kXOfgqSPM3 /tmp/tmp.sbKinxz6R8
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.EkRUQQ49Sv
++ mktemp
+ local LAST_ERR=/tmp/tmp.eGdfQQeVAV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EkRUQQ49Sv
+ cat /tmp/tmp.eGdfQQeVAV
+ rm /tmp/tmp.EkRUQQ49Sv /tmp/tmp.eGdfQQeVAV
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.wTFDqLsC95
++ mktemp
+ local LAST_ERR=/tmp/tmp.5zbS9vYpDM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wTFDqLsC95
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.5zbS9vYpDM
+ rm /tmp/tmp.wTFDqLsC95 /tmp/tmp.5zbS9vYpDM
+ return 0
+ check_crd_for_deletion PR-1736-b36c5346
+ local git_tag=PR-1736-b36c5346
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1736-b36c5346/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zfbx9DAi0x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7pvpuno2yA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zfbx9DAi0x
++ cat /tmp/tmp.7pvpuno2yA
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zfbx9DAi0x
++ cat /tmp/tmp.7pvpuno2yA
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.zfbx9DAi0x
++ cat /tmp/tmp.7pvpuno2yA
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.zfbx9DAi0x
++ cat /tmp/tmp.7pvpuno2yA
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.zfbx9DAi0x /tmp/tmp.7pvpuno2yA
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.bfurPlw6Hp
+ local LAST_OUT=/tmp/tmp.YiVasWKijF
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.2WCOMuIo50
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.pfAcR7CYm0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YiVasWKijF
+ cat /tmp/tmp.pfAcR7CYm0
+ rm /tmp/tmp.YiVasWKijF /tmp/tmp.pfAcR7CYm0
+ return 0
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "replset-overrides-28539" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bfurPlw6Hp
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.2WCOMuIo50
+ rm /tmp/tmp.bfurPlw6Hp /tmp/tmp.2WCOMuIo50
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.nNvuyHaV1E
++ mktemp
+ local LAST_ERR=/tmp/tmp.hQFsrjUnyN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nNvuyHaV1E
+ cat /tmp/tmp.hQFsrjUnyN
+ rm /tmp/tmp.nNvuyHaV1E /tmp/tmp.hQFsrjUnyN
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.bPMeikv0G7
++ mktemp
+ local LAST_ERR=/tmp/tmp.fVemjYW832
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bPMeikv0G7
namespace/psmdb-operator created
+ cat /tmp/tmp.fVemjYW832
+ rm /tmp/tmp.bPMeikv0G7 /tmp/tmp.fVemjYW832
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DaR8b2Jtkk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G2YoBzsTbc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DaR8b2Jtkk
++ cat /tmp/tmp.G2YoBzsTbc
++ rm /tmp/tmp.DaR8b2Jtkk /tmp/tmp.G2YoBzsTbc
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6WMa3OsC1L
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bf1VN6F7UK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6WMa3OsC1L
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10" modified.
+ cat /tmp/tmp.Bf1VN6F7UK
+ rm /tmp/tmp.6WMa3OsC1L /tmp/tmp.Bf1VN6F7UK
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.dtEqUMWNNw
++ mktemp
+ local LAST_ERR=/tmp/tmp.2E6ZSyRIaA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dtEqUMWNNw
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.2E6ZSyRIaA
+ rm /tmp/tmp.dtEqUMWNNw /tmp/tmp.2E6ZSyRIaA
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SGkKjWowHq
++ mktemp
+ local LAST_ERR=/tmp/tmp.sH5fmh6ivb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SGkKjWowHq
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.sH5fmh6ivb
+ rm /tmp/tmp.SGkKjWowHq /tmp/tmp.sH5fmh6ivb
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1736-b36c5346") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.OmVDKRpS28
++ mktemp
+ local LAST_ERR=/tmp/tmp.OLrzSg0yC9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OmVDKRpS28
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.OLrzSg0yC9
+ rm /tmp/tmp.OmVDKRpS28 /tmp/tmp.OLrzSg0yC9
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QDcPq7Y9fq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tN8iC3WXNc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QDcPq7Y9fq
++ cat /tmp/tmp.tN8iC3WXNc
++ rm /tmp/tmp.QDcPq7Y9fq /tmp/tmp.tN8iC3WXNc
++ return 0
+ wait_pod percona-server-mongodb-operator-86d54b85c6-8jf72
+ local pod=percona-server-mongodb-operator-86d54b85c6-8jf72
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-86d54b85c6-8jf72 to be ready.OK
+ create_namespace replset-overrides-23335
+ local namespace=replset-overrides-23335
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces replset-overrides-23335'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces replset-overrides-23335
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace replset-overrides-23335 --ignore-not-found
+ awk '{print$1}'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.gkhjAe13l0
++ mktemp
+ local LAST_OUT=/tmp/tmp.KDp7qS6EKV
+ local LAST_ERR=/tmp/tmp.WpCwHjwLmY
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.yydWTSIY4A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace replset-overrides-23335 --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gkhjAe13l0
+ cat /tmp/tmp.WpCwHjwLmY
+ rm /tmp/tmp.gkhjAe13l0 /tmp/tmp.WpCwHjwLmY
+ return 0
+ kubectl_bin wait --for=delete namespace replset-overrides-23335
++ mktemp
+ local LAST_OUT=/tmp/tmp.m1RsWUpJbR
++ mktemp
+ local LAST_ERR=/tmp/tmp.5zCjMaNzeI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace replset-overrides-23335
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KDp7qS6EKV
+ cat /tmp/tmp.yydWTSIY4A
+ rm /tmp/tmp.KDp7qS6EKV /tmp/tmp.yydWTSIY4A
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.m1RsWUpJbR
+ cat /tmp/tmp.5zCjMaNzeI
+ rm /tmp/tmp.m1RsWUpJbR /tmp/tmp.5zCjMaNzeI
+ return 0
+ desc 'create namespace replset-overrides-23335'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace replset-overrides-23335
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace replset-overrides-23335
++ mktemp
+ local LAST_OUT=/tmp/tmp.4oDifmnrC6
++ mktemp
+ local LAST_ERR=/tmp/tmp.0BTa7EYB24
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace replset-overrides-23335
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4oDifmnrC6
namespace/replset-overrides-23335 created
+ cat /tmp/tmp.0BTa7EYB24
+ rm /tmp/tmp.4oDifmnrC6 /tmp/tmp.0BTa7EYB24
+ return 0
++ kubectl_bin config current-context
+++ mktemp
namespace "gmp-system" deleted
++ local LAST_OUT=/tmp/tmp.bXGB2ByLXC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nXpsZeotLJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bXGB2ByLXC
++ cat /tmp/tmp.nXpsZeotLJ
++ rm /tmp/tmp.bXGB2ByLXC /tmp/tmp.nXpsZeotLJ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10 --namespace=replset-overrides-23335
++ mktemp
+ local LAST_OUT=/tmp/tmp.hUljkYNxBP
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wcme4qkqc1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10 --namespace=replset-overrides-23335
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hUljkYNxBP
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1736-b36c5346-8-cluster10" modified.
+ cat /tmp/tmp.Wcme4qkqc1
+ rm /tmp/tmp.hUljkYNxBP /tmp/tmp.Wcme4qkqc1
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.f1WoUlaU5E
++ mktemp
+ local LAST_ERR=/tmp/tmp.tiFPxhcmSM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f1WoUlaU5E
deployment.apps/psmdb-client created
+ cat /tmp/tmp.tiFPxhcmSM
+ rm /tmp/tmp.f1WoUlaU5E /tmp/tmp.tiFPxhcmSM
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Feb 26 11:12:39 2025
NAMESPACE: replset-overrides-23335
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.replset-overrides-23335.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace replset-overrides-23335 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace replset-overrides-23335

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace replset-overrides-23335 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace replset-overrides-23335 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xhajIiSm3Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A9h5cI20bN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xhajIiSm3Z
++ cat /tmp/tmp.A9h5cI20bN
++ rm /tmp/tmp.xhajIiSm3Z /tmp/tmp.A9h5cI20bN
++ return 0
+ MINIO_POD=minio-service-847fc8bb8d-bp2kj
+ wait_pod minio-service-847fc8bb8d-bp2kj
+ local pod=minio-service-847fc8bb8d-bp2kj
+ set +o xtrace
waiting for pod/minio-service-847fc8bb8d-bp2kj to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.replset-overrides-23335.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.3e3Up8mb0G
++ mktemp
+ local LAST_ERR=/tmp/tmp.URFUp8Yuxx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.replset-overrides-23335.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3e3Up8mb0G
service/minio-service created
+ cat /tmp/tmp.URFUp8Yuxx
+ rm /tmp/tmp.3e3Up8mb0G /tmp/tmp.URFUp8Yuxx
+ return 0
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.OlTZvt1VV4
++ mktemp
+ local LAST_ERR=/tmp/tmp.6UHCDtdRVs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OlTZvt1VV4
pod "aws-cli" deleted
+ cat /tmp/tmp.6UHCDtdRVs
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.OlTZvt1VV4 /tmp/tmp.6UHCDtdRVs
+ return 0
+ desc 'Case 1: Deploying a new cluster with hostname overrides'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 1: Deploying a new cluster with hostname overrides
-----------------------------------------------------------------------------------

+ test_deploy_with_host_overrides
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zgUZ0ar74z
++ mktemp
+ local LAST_ERR=/tmp/tmp.SfEOgYU88i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zgUZ0ar74z
secret/some-users created
secret/some-name-ssl created
secret/some-name-ssl-internal created
secret/minio-secret created
+ cat /tmp/tmp.SfEOgYU88i
+ rm /tmp/tmp.zgUZ0ar74z /tmp/tmp.SfEOgYU88i
+ return 0
+ echo 'creating external services'
creating external services
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/external-services.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nte9ikpFoO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vxy9a42jko
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/external-services.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nte9ikpFoO
service/external-rs0-0 created
service/external-rs0-1 created
service/external-rs0-2 created
+ cat /tmp/tmp.Vxy9a42jko
+ rm /tmp/tmp.nte9ikpFoO /tmp/tmp.Vxy9a42jko
+ return 0
+ echo 'creating PSMDB cluster: some-name'
creating PSMDB cluster: some-name
+ sed s/NAMESPACE/replset-overrides-23335/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name-overridden.yml
+ apply_cluster /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
+ cat /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1736-b36c5346"'
+ local LAST_OUT=/tmp/tmp.Kw98UjO6Ic
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:pbm-290"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.oVGhbBjdvr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Kw98UjO6Ic
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.oVGhbBjdvr
+ rm /tmp/tmp.Kw98UjO6Ic /tmp/tmp.oVGhbBjdvr
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qI6ypUl8uV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBYdSfvrEA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qI6ypUl8uV
++ cat /tmp/tmp.fBYdSfvrEA
++ rm /tmp/tmp.qI6ypUl8uV /tmp/tmp.fBYdSfvrEA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cdlK2MYtu2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lk1QTpH7z4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cdlK2MYtu2
++ cat /tmp/tmp.lk1QTpH7z4
++ rm /tmp/tmp.cdlK2MYtu2 /tmp/tmp.lk1QTpH7z4
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.....
+ echo 'writing some data'
writing some data
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OOfudf4xvn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6unReOu1px
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OOfudf4xvn
++ cat /tmp/tmp.6unReOu1px
++ rm /tmp/tmp.OOfudf4xvn /tmp/tmp.6unReOu1px
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.g8oYPDM0Bx
++ mktemp
+ local LAST_ERR=/tmp/tmp.7Pmg3lYnhM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.g8oYPDM0Bx
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://external-rs0-0.replset-overrides-23335.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("78b89356-dbdd-4922-947e-3460ac8acadd") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7Pmg3lYnhM
+ rm /tmp/tmp.g8oYPDM0Bx /tmp/tmp.7Pmg3lYnhM
+ return 0
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FlAqcSoxvj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3pLxI1R9x3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FlAqcSoxvj
++ cat /tmp/tmp.3pLxI1R9x3
++ rm /tmp/tmp.FlAqcSoxvj /tmp/tmp.3pLxI1R9x3
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ymhmbZIEm6
++ mktemp
+ local LAST_ERR=/tmp/tmp.kvBwWnuvOA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ymhmbZIEm6
+ cat /tmp/tmp.kvBwWnuvOA
+ rm /tmp/tmp.ymhmbZIEm6 /tmp/tmp.kvBwWnuvOA
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ run_backup minio backup-minio-physical physical
+ local storage=minio
+ local backup_name=backup-minio-physical
+ local type=physical
+ desc 'run backup backup-minio-physical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-physical
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup-minio-physical"
			| .spec.storageName = "minio"
			| .spec.type = "physical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/backup-minio.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.d8T3rlKxJC
++ mktemp
+ local LAST_ERR=/tmp/tmp.9yURbTw992
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.d8T3rlKxJC
perconaservermongodbbackup.psmdb.percona.com/backup-minio-physical created
+ cat /tmp/tmp.9yURbTw992
+ rm /tmp/tmp.d8T3rlKxJC /tmp/tmp.9yURbTw992
+ return 0
+ wait_backup backup-minio-physical
+ local backup_name=backup-minio-physical
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-physical to reach ready state..................
+ run_recovery_check some-name backup-minio-physical external-rs0-0.replset-overrides-23335
+ local cluster=some-name
+ local backup=backup-minio-physical
+ local host=external-rs0-0.replset-overrides-23335
+ echo 'write data after backup'
write data after backup
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NHRO2zpFvV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HCEHGRmBys
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NHRO2zpFvV
++ cat /tmp/tmp.HCEHGRmBys
++ rm /tmp/tmp.NHRO2zpFvV /tmp/tmp.HCEHGRmBys
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gg5MsBay1o
++ mktemp
+ local LAST_ERR=/tmp/tmp.40NR1y58ap
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Gg5MsBay1o
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://external-rs0-0.replset-overrides-23335.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a15da51e-0700-4e2d-85cb-f9f2d9cd5b85") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.40NR1y58ap
+ rm /tmp/tmp.Gg5MsBay1o /tmp/tmp.40NR1y58ap
+ return 0
+ echo 'restore to backup backup-minio-physical'
restore to backup backup-minio-physical
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 -2nd
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6GEimKXXT9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VwwqlDfzZ7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6GEimKXXT9
++ cat /tmp/tmp.VwwqlDfzZ7
++ rm /tmp/tmp.6GEimKXXT9 /tmp/tmp.VwwqlDfzZ7
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.k7xggYYpUE
++ mktemp
+ local LAST_ERR=/tmp/tmp.WDDXgck8Rh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k7xggYYpUE
+ cat /tmp/tmp.WDDXgck8Rh
+ rm /tmp/tmp.k7xggYYpUE /tmp/tmp.WDDXgck8Rh
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find-2nd.json /tmp/tmp.rjYAYV6OUn/find-2nd
+ run_restore backup-minio-physical
+ local backup_name=backup-minio-physical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-physical/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-physical/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.J64YDq9TEH
++ mktemp
+ local LAST_ERR=/tmp/tmp.TqzVeSH1f4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J64YDq9TEH
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-physical created
+ cat /tmp/tmp.TqzVeSH1f4
+ rm /tmp/tmp.J64YDq9TEH /tmp/tmp.TqzVeSH1f4
+ return 0
+ wait_restore backup-minio-physical some-name
+ local backup_name=backup-minio-physical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ set +o xtrace
waiting psmdb-restore/restore-backup-minio-physical to reach ready state..............................................................................................................................................................................................
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MkTTNDacZN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gt2MfHOg7F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MkTTNDacZN
++ cat /tmp/tmp.gt2MfHOg7F
++ rm /tmp/tmp.MkTTNDacZN /tmp/tmp.gt2MfHOg7F
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cGNF5Ad7pU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SzBIONXnGv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cGNF5Ad7pU
++ cat /tmp/tmp.SzBIONXnGv
++ rm /tmp/tmp.cGNF5Ad7pU /tmp/tmp.SzBIONXnGv
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.luB3UBKFQc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oLfCOitQMV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.luB3UBKFQc
++ cat /tmp/tmp.oLfCOitQMV
++ rm /tmp/tmp.luB3UBKFQc /tmp/tmp.oLfCOitQMV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lqU5OdhA6G
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2BuiyHvibt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lqU5OdhA6G
++ cat /tmp/tmp.2BuiyHvibt
++ rm /tmp/tmp.lqU5OdhA6G /tmp/tmp.2BuiyHvibt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GEKKiQy9rW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CtQotK9lta
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GEKKiQy9rW
++ cat /tmp/tmp.CtQotK9lta
++ rm /tmp/tmp.GEKKiQy9rW /tmp/tmp.CtQotK9lta
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nfxiCwk2aA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xgPlmmnexm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nfxiCwk2aA
++ cat /tmp/tmp.xgPlmmnexm
++ rm /tmp/tmp.nfxiCwk2aA /tmp/tmp.xgPlmmnexm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AWyuawDJCp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Gbslk3DAcs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AWyuawDJCp
++ cat /tmp/tmp.Gbslk3DAcs
++ rm /tmp/tmp.AWyuawDJCp /tmp/tmp.Gbslk3DAcs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.It14qiieS4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NNLYNOuBf4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.It14qiieS4
++ cat /tmp/tmp.NNLYNOuBf4
++ rm /tmp/tmp.It14qiieS4 /tmp/tmp.NNLYNOuBf4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gx8MAyU757
+++ mktemp
++ local LAST_ERR=/tmp/tmp.np1ldXZKHn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gx8MAyU757
++ cat /tmp/tmp.np1ldXZKHn
++ rm /tmp/tmp.gx8MAyU757 /tmp/tmp.np1ldXZKHn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XF6JCNq28H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gduTC1N0kL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XF6JCNq28H
++ cat /tmp/tmp.gduTC1N0kL
++ rm /tmp/tmp.XF6JCNq28H /tmp/tmp.gduTC1N0kL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.drb0D8SQxx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ASbOafC5UV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.drb0D8SQxx
++ cat /tmp/tmp.ASbOafC5UV
++ rm /tmp/tmp.drb0D8SQxx /tmp/tmp.ASbOafC5UV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tgBgP5jOa4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uuYIw5R10S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tgBgP5jOa4
++ cat /tmp/tmp.uuYIw5R10S
++ rm /tmp/tmp.tgBgP5jOa4 /tmp/tmp.uuYIw5R10S
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J8Doc9bHJu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LXqta1epmJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J8Doc9bHJu
++ cat /tmp/tmp.LXqta1epmJ
++ rm /tmp/tmp.J8Doc9bHJu /tmp/tmp.LXqta1epmJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Gm3dAmxDXX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.91RlQ1gpXt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Gm3dAmxDXX
++ cat /tmp/tmp.91RlQ1gpXt
++ rm /tmp/tmp.Gm3dAmxDXX /tmp/tmp.91RlQ1gpXt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zxYJyEwXxq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ptzJYiYhxM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zxYJyEwXxq
++ cat /tmp/tmp.ptzJYiYhxM
++ rm /tmp/tmp.zxYJyEwXxq /tmp/tmp.ptzJYiYhxM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5ejEtW18a2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3fbPB6Plui
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5ejEtW18a2
++ cat /tmp/tmp.3fbPB6Plui
++ rm /tmp/tmp.5ejEtW18a2 /tmp/tmp.3fbPB6Plui
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GEWdNE4Lrs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tmGCUUTEgv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GEWdNE4Lrs
++ cat /tmp/tmp.tmGCUUTEgv
++ rm /tmp/tmp.GEWdNE4Lrs /tmp/tmp.tmGCUUTEgv
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0W2PD9vuMj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yz8bFiBTXk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0W2PD9vuMj
++ cat /tmp/tmp.Yz8bFiBTXk
++ rm /tmp/tmp.0W2PD9vuMj /tmp/tmp.Yz8bFiBTXk
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ci8jx7Htk9
++ mktemp
+ local LAST_ERR=/tmp/tmp.qEdPmOE4JL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ci8jx7Htk9
+ cat /tmp/tmp.qEdPmOE4JL
+ rm /tmp/tmp.Ci8jx7Htk9 /tmp/tmp.qEdPmOE4JL
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.289JgCgLqM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fER9M0C7Vv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.289JgCgLqM
++ cat /tmp/tmp.fER9M0C7Vv
++ rm /tmp/tmp.289JgCgLqM /tmp/tmp.fER9M0C7Vv
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ delete_cluster some-name
+ local cluster=some-name
+ echo 'deleting cluster: some-name'
deleting cluster: some-name
+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.3L3gZ7NBhK
++ mktemp
+ local LAST_ERR=/tmp/tmp.iLQ5RzaCny
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3L3gZ7NBhK
perconaservermongodb.psmdb.percona.com "some-name" deleted
+ cat /tmp/tmp.iLQ5RzaCny
+ rm /tmp/tmp.3L3gZ7NBhK /tmp/tmp.iLQ5RzaCny
+ return 0
+ wait_for_delete psmdb/some-name
+ local res=psmdb/some-name
+ local wait_time=60
+ set +o xtrace
waiting for psmdb/some-name to be deletedError from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ wait_for_delete pod/some-name-rs0-0
+ local res=pod/some-name-rs0-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be deleted.Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
+ kubectl delete secrets --all
secret "internal-some-name-users" deleted
secret "minio-secret" deleted
secret "minio-service" deleted
secret "sh.helm.release.v1.minio-service.v1" deleted
secret "some-name-mongodb-encryption-key" deleted
secret "some-name-pbm-config" deleted
secret "some-name-ssl" deleted
secret "some-name-ssl-internal" deleted
secret "some-users" deleted
+ desc 'Case 1: PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 1: PASSED
-----------------------------------------------------------------------------------

+ desc 'Case 2: Patching a running cluster to override hostnames'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 2: Patching a running cluster to override hostnames
-----------------------------------------------------------------------------------

+ test_override_host_after_deploy
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ve7NjXIfC4
++ mktemp
+ local LAST_ERR=/tmp/tmp.IevPJWoaPn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ve7NjXIfC4
secret/some-users created
secret/some-name-ssl created
secret/some-name-ssl-internal created
secret/minio-secret created
+ cat /tmp/tmp.IevPJWoaPn
+ rm /tmp/tmp.ve7NjXIfC4 /tmp/tmp.IevPJWoaPn
+ return 0
+ echo 'creating PSMDB cluster: some-name'
creating PSMDB cluster: some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:pbm-290"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.VK3nEz7h2v
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1736-b36c5346"'
+ local LAST_ERR=/tmp/tmp.Pnn3x8nP6i
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VK3nEz7h2v
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Pnn3x8nP6i
+ rm /tmp/tmp.VK3nEz7h2v /tmp/tmp.Pnn3x8nP6i
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4L0Xzda646
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eL0EfijOSj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4L0Xzda646
++ cat /tmp/tmp.eL0EfijOSj
++ rm /tmp/tmp.4L0Xzda646 /tmp/tmp.eL0EfijOSj
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.87bKH2haUP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UWLJvyWUJ6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.87bKH2haUP
++ cat /tmp/tmp.UWLJvyWUJ6
++ rm /tmp/tmp.87bKH2haUP /tmp/tmp.UWLJvyWUJ6
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness......
+ sleep 10
+ echo 'writing some data'
writing some data
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OHtlQ2hby9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sMhPhZkwWR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OHtlQ2hby9
++ cat /tmp/tmp.sMhPhZkwWR
++ rm /tmp/tmp.OHtlQ2hby9 /tmp/tmp.sMhPhZkwWR
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NRitvsWVhl
++ mktemp
+ local LAST_ERR=/tmp/tmp.UOYvslngJN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NRitvsWVhl
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d8dd46b6-754f-49ac-b143-0f263043aaa7") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.UOYvslngJN
+ rm /tmp/tmp.NRitvsWVhl /tmp/tmp.UOYvslngJN
+ return 0
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3h1ccplm1f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UYDjT2usuR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3h1ccplm1f
++ cat /tmp/tmp.UYDjT2usuR
++ rm /tmp/tmp.3h1ccplm1f /tmp/tmp.UYDjT2usuR
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IY3wVrcvVy
++ mktemp
+ local LAST_ERR=/tmp/tmp.yuDBDg6tx5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IY3wVrcvVy
+ cat /tmp/tmp.yuDBDg6tx5
+ rm /tmp/tmp.IY3wVrcvVy /tmp/tmp.yuDBDg6tx5
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ run_backup minio backup-minio-logical
+ local storage=minio
+ local backup_name=backup-minio-logical
+ local type=logical
+ desc 'run backup backup-minio-logical'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-logical
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup-minio-logical"
			| .spec.storageName = "minio"
			| .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/backup-minio.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.udeDQHzCwx
++ mktemp
+ local LAST_ERR=/tmp/tmp.FDxPtqrrGM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.udeDQHzCwx
perconaservermongodbbackup.psmdb.percona.com/backup-minio-logical created
+ cat /tmp/tmp.FDxPtqrrGM
+ rm /tmp/tmp.udeDQHzCwx /tmp/tmp.FDxPtqrrGM
+ return 0
+ wait_backup backup-minio-logical
+ local backup_name=backup-minio-logical
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-logical to reach ready state..................
+ echo 'creating external services'
creating external services
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/external-services.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.i1GvyUNYX8
++ mktemp
+ local LAST_ERR=/tmp/tmp.xvct6jFAh0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/external-services.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.i1GvyUNYX8
service/external-rs0-0 unchanged
service/external-rs0-1 unchanged
service/external-rs0-2 unchanged
+ cat /tmp/tmp.xvct6jFAh0
+ rm /tmp/tmp.i1GvyUNYX8 /tmp/tmp.xvct6jFAh0
+ return 0
+ echo 'patching PSMDB cluster with replsetOverrides: some-name'
patching PSMDB cluster with replsetOverrides: some-name
+ sed s/NAMESPACE/replset-overrides-23335/g /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name-overridden.yml
+ apply_cluster /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
+ '[' -z '' ']'
+ cat_config /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
+ kubectl_bin apply -f -
+ cat /tmp/tmp.rjYAYV6OUn/some-name-overridden.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:pbm-290"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1736-b36c5346"'
+ local LAST_OUT=/tmp/tmp.G5StT0CGxQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.fstOUXBBwb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G5StT0CGxQ
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.fstOUXBBwb
+ rm /tmp/tmp.G5StT0CGxQ /tmp/tmp.fstOUXBBwb
+ return 0
+ wait_for_cluster_state some-name ready
+ local cluster_name=some-name
+ local target_state=ready
+ echo -n 'Waiting for cluster to reach ready state'
Waiting for cluster to reach ready state+ local timeout=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qcaIyBUQc6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4A4LUX6luT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qcaIyBUQc6
++ cat /tmp/tmp.4A4LUX6luT
++ rm /tmp/tmp.qcaIyBUQc6 /tmp/tmp.4A4LUX6luT
++ return 0
+ [[ ready == ready ]]
+ echo

+ echo 'checking if we can read existing data'
checking if we can read existing data
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eiKHNElUmf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uzoF3ZKJ2U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eiKHNElUmf
++ cat /tmp/tmp.uzoF3ZKJ2U
++ rm /tmp/tmp.eiKHNElUmf /tmp/tmp.uzoF3ZKJ2U
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TZhYS79aa3
++ mktemp
+ local LAST_ERR=/tmp/tmp.RZXF0zKyzY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TZhYS79aa3
+ cat /tmp/tmp.RZXF0zKyzY
+ rm /tmp/tmp.TZhYS79aa3 /tmp/tmp.RZXF0zKyzY
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ run_recovery_check some-name backup-minio-logical external-rs0-0.replset-overrides-23335
+ local cluster=some-name
+ local backup=backup-minio-logical
+ local host=external-rs0-0.replset-overrides-23335
+ echo 'write data after backup'
write data after backup
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.io4wj1aFcN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.33RcitLqbG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.io4wj1aFcN
++ cat /tmp/tmp.33RcitLqbG
++ rm /tmp/tmp.io4wj1aFcN /tmp/tmp.33RcitLqbG
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.luiG6y86cE
++ mktemp
+ local LAST_ERR=/tmp/tmp.BzyzDWnK6q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.luiG6y86cE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://external-rs0-0.replset-overrides-23335.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("67a54045-54c1-4ef5-8e30-29a2f85875d5") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.BzyzDWnK6q
+ rm /tmp/tmp.luiG6y86cE /tmp/tmp.BzyzDWnK6q
+ return 0
+ echo 'restore to backup backup-minio-logical'
restore to backup backup-minio-logical
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 -2nd
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1ydhbQQ29V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4Etyd7xqJJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1ydhbQQ29V
++ cat /tmp/tmp.4Etyd7xqJJ
++ rm /tmp/tmp.1ydhbQQ29V /tmp/tmp.4Etyd7xqJJ
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SJlvep1Bsc
++ mktemp
+ local LAST_ERR=/tmp/tmp.cRGAH8MOBl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SJlvep1Bsc
+ cat /tmp/tmp.cRGAH8MOBl
+ rm /tmp/tmp.SJlvep1Bsc /tmp/tmp.cRGAH8MOBl
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find-2nd.json /tmp/tmp.rjYAYV6OUn/find-2nd
+ run_restore backup-minio-logical
+ local backup_name=backup-minio-logical
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-logical/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-logical/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.GAyD1d3LX3
++ mktemp
+ local LAST_ERR=/tmp/tmp.u3yHtKCyEZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GAyD1d3LX3
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-logical created
+ cat /tmp/tmp.u3yHtKCyEZ
+ rm /tmp/tmp.GAyD1d3LX3 /tmp/tmp.u3yHtKCyEZ
+ return 0
+ wait_restore backup-minio-logical some-name
+ local backup_name=backup-minio-logical
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ set +o xtrace
waiting psmdb-restore/restore-backup-minio-logical to reach ready state......................
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2vABxHLj03
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5RtVhAsTCl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2vABxHLj03
++ cat /tmp/tmp.5RtVhAsTCl
++ rm /tmp/tmp.2vABxHLj03 /tmp/tmp.5RtVhAsTCl
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oc7sIHuhDx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ppUddPv4F4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Oc7sIHuhDx
++ cat /tmp/tmp.ppUddPv4F4
++ rm /tmp/tmp.Oc7sIHuhDx /tmp/tmp.ppUddPv4F4
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.v4mWEYcS3x
++ mktemp
+ local LAST_ERR=/tmp/tmp.rB5Y4sEDfj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@external-rs0-0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v4mWEYcS3x
+ cat /tmp/tmp.rB5Y4sEDfj
+ rm /tmp/tmp.v4mWEYcS3x /tmp/tmp.rB5Y4sEDfj
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L7unoTX9zj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QS8SVGB1Kk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L7unoTX9zj
++ cat /tmp/tmp.QS8SVGB1Kk
++ rm /tmp/tmp.L7unoTX9zj /tmp/tmp.QS8SVGB1Kk
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ delete_cluster some-name
+ local cluster=some-name
+ echo 'deleting cluster: some-name'
deleting cluster: some-name
+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.lL53c9664G
++ mktemp
+ local LAST_ERR=/tmp/tmp.mXJ2wMsCz0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lL53c9664G
perconaservermongodb.psmdb.percona.com "some-name" deleted
+ cat /tmp/tmp.mXJ2wMsCz0
+ rm /tmp/tmp.lL53c9664G /tmp/tmp.mXJ2wMsCz0
+ return 0
+ wait_for_delete psmdb/some-name
+ local res=psmdb/some-name
+ local wait_time=60
+ set +o xtrace
waiting for psmdb/some-name to be deletedError from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ wait_for_delete pod/some-name-rs0-0
+ local res=pod/some-name-rs0-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be deleted..Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
+ kubectl delete secrets --all
secret "internal-some-name-users" deleted
secret "minio-secret" deleted
secret "some-name-mongodb-encryption-key" deleted
secret "some-name-ssl" deleted
secret "some-name-ssl-internal" deleted
secret "some-users" deleted
+ desc 'Case 2: PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 2: PASSED
-----------------------------------------------------------------------------------

+ desc 'Case 3: Overriding member priorities'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 3: Overriding member priorities
-----------------------------------------------------------------------------------

+ test_override_priority
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Idu95qihII
++ mktemp
+ local LAST_ERR=/tmp/tmp.HUVkpM2YPC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/secrets_with_tls.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Idu95qihII
secret/some-users created
secret/some-name-ssl created
secret/some-name-ssl-internal created
secret/minio-secret created
+ cat /tmp/tmp.HUVkpM2YPC
+ rm /tmp/tmp.Idu95qihII /tmp/tmp.HUVkpM2YPC
+ return 0
+ echo 'creating PSMDB cluster: some-name'
creating PSMDB cluster: some-name
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name-override-priority.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name-override-priority.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/conf/some-name-override-priority.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1736-b36c5346"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:pbm-290"'
+ local LAST_OUT=/tmp/tmp.J9HQFRpavx
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.vXCQwFIf12
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J9HQFRpavx
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.vXCQwFIf12
+ rm /tmp/tmp.J9HQFRpavx /tmp/tmp.vXCQwFIf12
+ return 0
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QFJ1cytfM5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xv2bwTwglT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QFJ1cytfM5
++ cat /tmp/tmp.Xv2bwTwglT
++ rm /tmp/tmp.QFJ1cytfM5 /tmp/tmp.Xv2bwTwglT
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NLSI0RgCUQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7L1eDu1GRo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NLSI0RgCUQ
++ cat /tmp/tmp.7L1eDu1GRo
++ rm /tmp/tmp.NLSI0RgCUQ /tmp/tmp.7L1eDu1GRo
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.....
+ echo 'writing some data'
writing some data
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fZcJShiz0A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JD7ReyaTxq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fZcJShiz0A
++ cat /tmp/tmp.JD7ReyaTxq
++ rm /tmp/tmp.fZcJShiz0A /tmp/tmp.JD7ReyaTxq
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.JAKChf9ACU
++ mktemp
+ local LAST_ERR=/tmp/tmp.LwWXudeN78
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JAKChf9ACU
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.replset-overrides-23335.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("9c9355d0-71ae-4aed-b4e8-5e20337b8c26") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.LwWXudeN78
+ rm /tmp/tmp.JAKChf9ACU /tmp/tmp.LwWXudeN78
+ return 0
+ compare_mongo_cmd find databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local command=find
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KPaN1xtJeo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WGGvyN9TWY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KPaN1xtJeo
++ cat /tmp/tmp.WGGvyN9TWY
++ rm /tmp/tmp.KPaN1xtJeo /tmp/tmp.WGGvyN9TWY
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.4Gn8NXO5nY
++ mktemp
+ local LAST_ERR=/tmp/tmp.1LNOquRO4b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4Gn8NXO5nY
+ cat /tmp/tmp.1LNOquRO4b
+ rm /tmp/tmp.4Gn8NXO5nY /tmp/tmp.1LNOquRO4b
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/find.json /tmp/tmp.rjYAYV6OUn/find
+ echo 'checking member priorities'
checking member priorities
+ run_mongo 'rs.conf().members.map(m => m.priority)' databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local 'command=rs.conf().members.map(m => m.priority)'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b8Mp1Txbpt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0Z6iIdPA4Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b8Mp1Txbpt
++ cat /tmp/tmp.0Z6iIdPA4Y
++ rm /tmp/tmp.b8Mp1Txbpt /tmp/tmp.0Z6iIdPA4Y
++ return 0
+ local client_container=psmdb-client-874f474b6-c2rkl
+ local mongo_flag=
+ [[ databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''rs.conf().members.map(m => m.priority)\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.fdXQv6GGit
++ mktemp
+ local LAST_ERR=/tmp/tmp.TcF1qh2zH1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c2rkl -- bash -c 'printf '\''rs.conf().members.map(m => m.priority)\n'\'' | mongo mongodb+srv://databaseAdmin:databaseAdmin123456@some-name-rs0.replset-overrides-23335.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fdXQv6GGit
+ cat /tmp/tmp.TcF1qh2zH1
+ rm /tmp/tmp.fdXQv6GGit /tmp/tmp.TcF1qh2zH1
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1736/e2e-tests/replset-overrides/compare/priorities.json /tmp/tmp.rjYAYV6OUn/priorities.json
+ echo 'member priorities are OK'
member priorities are OK
+ delete_cluster some-name
+ local cluster=some-name
+ echo 'deleting cluster: some-name'
deleting cluster: some-name
+ kubectl_bin delete psmdb some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.QcmYAlqc6e
++ mktemp
+ local LAST_ERR=/tmp/tmp.NKvemaWVQx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb some-name
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QcmYAlqc6e
perconaservermongodb.psmdb.percona.com "some-name" deleted
+ cat /tmp/tmp.NKvemaWVQx
+ rm /tmp/tmp.QcmYAlqc6e /tmp/tmp.NKvemaWVQx
+ return 0
+ wait_for_delete psmdb/some-name
+ local res=psmdb/some-name
+ local wait_time=60
+ set +o xtrace
waiting for psmdb/some-name to be deletedError from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
Error from server (NotFound): perconaservermongodbs.psmdb.percona.com "some-name" not found
+ wait_for_delete pod/some-name-rs0-0
+ local res=pod/some-name-rs0-0
+ local wait_time=60
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be deletedError from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
Error from server (NotFound): pods "some-name-rs0-0" not found
+ kubectl delete secrets --all
secret "internal-some-name-users" deleted
secret "minio-secret" deleted
secret "some-name-mongodb-encryption-key" deleted
secret "some-name-ssl" deleted
secret "some-name-ssl-internal" deleted
secret "some-users" deleted
+ desc 'Case 3: PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
Case 3: PASSED
-----------------------------------------------------------------------------------

+ desc 'All cases PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
All cases PASSED
-----------------------------------------------------------------------------------