Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/logs/pvc-auto-resize.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ '[' 0 == 1 ']'
+ ensure_default_sc_allows_expansion
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Hc3lDklAPp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QTw2rPUaZG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Hc3lDklAPp
++ cat /tmp/tmp.QTw2rPUaZG
++ rm /tmp/tmp.Hc3lDklAPp /tmp/tmp.QTw2rPUaZG
++ return 0
+ default_sc=standard-rwo
+ echo 'Checking if default storageclass standard-rwo allows volume expansion'
Checking if default storageclass standard-rwo allows volume expansion
+ local allowVolumeExpansion
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.name=="standard-rwo")].allowVolumeExpansion}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LJcxI8gq8x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mmVRGhCVqJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.name=="standard-rwo")].allowVolumeExpansion}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LJcxI8gq8x
++ cat /tmp/tmp.mmVRGhCVqJ
++ rm /tmp/tmp.LJcxI8gq8x /tmp/tmp.mmVRGhCVqJ
++ return 0
+ allowVolumeExpansion=true
+ [[ true != \t\r\u\e ]]
+ create_infra pvc-auto-resize-18540
+ local ns=pvc-auto-resize-18540
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.ENV4dB1rUN
++ mktemp
+ local LAST_ERR=/tmp/tmp.6Q9oXbfcfV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ENV4dB1rUN
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.6Q9oXbfcfV
+ rm /tmp/tmp.ENV4dB1rUN /tmp/tmp.6Q9oXbfcfV
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.pnYH1Wzrdw
++ mktemp
+ local LAST_ERR=/tmp/tmp.n2gExYKKdB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pnYH1Wzrdw
+ cat /tmp/tmp.n2gExYKKdB
+ rm /tmp/tmp.pnYH1Wzrdw /tmp/tmp.n2gExYKKdB
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8al5eRp32p
++ mktemp
+ local LAST_ERR=/tmp/tmp.0vxyiFTfV5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8al5eRp32p
+ cat /tmp/tmp.0vxyiFTfV5
+ rm /tmp/tmp.8al5eRp32p /tmp/tmp.0vxyiFTfV5
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n pvc-auto-resize-16116 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.cEWnuMbdCp
++ mktemp
+ local LAST_ERR=/tmp/tmp.WSGdKd2NQ7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cEWnuMbdCp
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.WSGdKd2NQ7
+ rm /tmp/tmp.cEWnuMbdCp /tmp/tmp.WSGdKd2NQ7
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.UKcZ4lDUs2
++ mktemp
+ local LAST_ERR=/tmp/tmp.LTEh8vmkoL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UKcZ4lDUs2
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.LTEh8vmkoL
+ rm /tmp/tmp.UKcZ4lDUs2 /tmp/tmp.LTEh8vmkoL
+ return 0
+ check_crd_for_deletion PR-2227-94dbdcd6
+ local git_tag=PR-2227-94dbdcd6
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2227-94dbdcd6/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LC54nPiUiB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1MZ6gOInnb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LC54nPiUiB
++ cat /tmp/tmp.1MZ6gOInnb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LC54nPiUiB
++ cat /tmp/tmp.1MZ6gOInnb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.LC54nPiUiB
++ cat /tmp/tmp.1MZ6gOInnb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.LC54nPiUiB
++ cat /tmp/tmp.1MZ6gOInnb
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.LC54nPiUiB /tmp/tmp.1MZ6gOInnb
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.mjfzl4PBFR
+ local LAST_OUT=/tmp/tmp.Mz6zD88OlT
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.TXXHxtTAna
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.xND4GxkbBJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mjfzl4PBFR
+ cat /tmp/tmp.xND4GxkbBJ
+ rm /tmp/tmp.mjfzl4PBFR /tmp/tmp.xND4GxkbBJ
+ return 0
namespace "pvc-auto-resize-16116" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Mz6zD88OlT
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.TXXHxtTAna
+ rm /tmp/tmp.Mz6zD88OlT /tmp/tmp.TXXHxtTAna
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.AXBcmv4qkn
++ mktemp
+ local LAST_ERR=/tmp/tmp.KaZPKhxo38
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AXBcmv4qkn
+ cat /tmp/tmp.KaZPKhxo38
+ rm /tmp/tmp.AXBcmv4qkn /tmp/tmp.KaZPKhxo38
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.KwoYkJaaId
++ mktemp
+ local LAST_ERR=/tmp/tmp.H0mqTHcB0W
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KwoYkJaaId
namespace/psmdb-operator created
+ cat /tmp/tmp.H0mqTHcB0W
+ rm /tmp/tmp.KwoYkJaaId /tmp/tmp.H0mqTHcB0W
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BLKxfr4FuW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pmgJGuBAVy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BLKxfr4FuW
++ cat /tmp/tmp.pmgJGuBAVy
++ rm /tmp/tmp.BLKxfr4FuW /tmp/tmp.pmgJGuBAVy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qoskpbCUkX
++ mktemp
+ local LAST_ERR=/tmp/tmp.0FtxfOOWXo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qoskpbCUkX
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15" modified.
+ cat /tmp/tmp.0FtxfOOWXo
+ rm /tmp/tmp.qoskpbCUkX /tmp/tmp.0FtxfOOWXo
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2227-94dbdcd6'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2227-94dbdcd6
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FLGFsvlaG7
++ mktemp
+ local LAST_ERR=/tmp/tmp.dHRlykv9r5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FLGFsvlaG7
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.dHRlykv9r5
+ rm /tmp/tmp.FLGFsvlaG7 /tmp/tmp.dHRlykv9r5
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XDl8bof8ye
++ mktemp
+ local LAST_ERR=/tmp/tmp.WbZEZs1fuA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XDl8bof8ye
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.WbZEZs1fuA
+ rm /tmp/tmp.XDl8bof8ye /tmp/tmp.WbZEZs1fuA
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2227-94dbdcd6") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.mxPyu2an9w
++ mktemp
+ local LAST_ERR=/tmp/tmp.TU48qv4YTm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mxPyu2an9w
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.TU48qv4YTm
+ rm /tmp/tmp.mxPyu2an9w /tmp/tmp.TU48qv4YTm
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7jDFYtSzCz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P1Vs1Gctp5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7jDFYtSzCz
++ cat /tmp/tmp.P1Vs1Gctp5
++ rm /tmp/tmp.7jDFYtSzCz /tmp/tmp.P1Vs1Gctp5
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-84cfb447fb-wwnhw
+ local pod=percona-server-mongodb-operator-84cfb447fb-wwnhw
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-84cfb447fb-wwnhw to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tS9cYxORxk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aNbeRMBpxp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tS9cYxORxk
++ cat /tmp/tmp.aNbeRMBpxp
++ rm /tmp/tmp.tS9cYxORxk /tmp/tmp.aNbeRMBpxp
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-84cfb447fb-wwnhw
++ mktemp
+ local LAST_OUT=/tmp/tmp.z7trhD8XlN
++ mktemp
+ local LAST_ERR=/tmp/tmp.xGCRYvEgkC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-84cfb447fb-wwnhw
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z7trhD8XlN
+ cat /tmp/tmp.xGCRYvEgkC
+ rm /tmp/tmp.z7trhD8XlN /tmp/tmp.xGCRYvEgkC
+ return 0
2026-02-03T01:48:56.529Z	INFO	setup	Manager starting up	{"gitCommit": "94dbdcd604e08c657ed02d302eed8841e81611a6", "gitBranch": "PR-2227-94dbdcd6", "buildTime": "", "goVersion": "go1.25.6", "os": "linux", "arch": "amd64"}
+ create_namespace pvc-auto-resize-18540
+ local namespace=pvc-auto-resize-18540
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pvc-auto-resize-18540'
+ awk '{print$1}'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pvc-auto-resize-18540
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace pvc-auto-resize-18540 --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.CeOoGa9oEh
+ local LAST_OUT=/tmp/tmp.K7KSCFP5Tp
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.f9MzKJFLVt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.SmQSLDeAC2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pvc-auto-resize-18540 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CeOoGa9oEh
+ cat /tmp/tmp.f9MzKJFLVt
+ rm /tmp/tmp.CeOoGa9oEh /tmp/tmp.f9MzKJFLVt
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.K7KSCFP5Tp
+ cat /tmp/tmp.SmQSLDeAC2
+ rm /tmp/tmp.K7KSCFP5Tp /tmp/tmp.SmQSLDeAC2
+ return 0
+ kubectl_bin wait --for=delete namespace pvc-auto-resize-18540
++ mktemp
+ local LAST_OUT=/tmp/tmp.DMS5fF89VO
++ mktemp
+ local LAST_ERR=/tmp/tmp.0OnsXG9tUG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pvc-auto-resize-18540
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DMS5fF89VO
+ cat /tmp/tmp.0OnsXG9tUG
+ rm /tmp/tmp.DMS5fF89VO /tmp/tmp.0OnsXG9tUG
+ return 0
+ desc 'create namespace pvc-auto-resize-18540'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pvc-auto-resize-18540
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pvc-auto-resize-18540
++ mktemp
+ local LAST_OUT=/tmp/tmp.LbEMGOVj68
++ mktemp
+ local LAST_ERR=/tmp/tmp.Di07j1H5nh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pvc-auto-resize-18540
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LbEMGOVj68
namespace/pvc-auto-resize-18540 created
+ cat /tmp/tmp.Di07j1H5nh
+ rm /tmp/tmp.LbEMGOVj68 /tmp/tmp.Di07j1H5nh
+ return 0
+ set_kube_ctx pvc-auto-resize-18540
+ local namespace=pvc-auto-resize-18540
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kIYdHKcPwx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H5ouMQnMb9
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kIYdHKcPwx
++ cat /tmp/tmp.H5ouMQnMb9
++ rm /tmp/tmp.kIYdHKcPwx /tmp/tmp.H5ouMQnMb9
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15 --namespace=pvc-auto-resize-18540
++ mktemp
+ local LAST_OUT=/tmp/tmp.z0pAXlxtXE
++ mktemp
+ local LAST_ERR=/tmp/tmp.pJk8F29rtY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15 --namespace=pvc-auto-resize-18540
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z0pAXlxtXE
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2227-94dbdcd6-1-cluster15" modified.
+ cat /tmp/tmp.pJk8F29rtY
+ rm /tmp/tmp.z0pAXlxtXE /tmp/tmp.pJk8F29rtY
+ return 0
+ desc 'create secrets and psmdb client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and psmdb client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hBmSJMyYqz
++ mktemp
+ local LAST_ERR=/tmp/tmp.LZLVgoChFM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hBmSJMyYqz
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.LZLVgoChFM
+ rm /tmp/tmp.hBmSJMyYqz /tmp/tmp.LZLVgoChFM
+ return 0
+ desc 'create PSMDB cluster with storageAutoscaling enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PSMDB cluster with storageAutoscaling enabled
-----------------------------------------------------------------------------------

+ cluster=some-name
+ spinup_psmdb some-name-rs0 /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ local cluster=some-name-rs0
+ local config=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ local size=3
+ desc 'create first PSMDB cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ local LAST_OUT=/tmp/tmp.CJs9XcsSJo
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/pvc-auto-resize-18540/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2227-94dbdcd6"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.7PDr6wLfTR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CJs9XcsSJo
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.7PDr6wLfTR
+ rm /tmp/tmp.CJs9XcsSJo /tmp/tmp.7PDr6wLfTR
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready...............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OCSmvShBfm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0jOcAbcO2V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OCSmvShBfm
++ cat /tmp/tmp.0jOcAbcO2V
++ rm /tmp/tmp.OCSmvShBfm /tmp/tmp.0jOcAbcO2V
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P0B4y0e91C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tcd60mu356
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P0B4y0e91C
++ cat /tmp/tmp.Tcd60mu356
++ rm /tmp/tmp.P0B4y0e91C /tmp/tmp.Tcd60mu356
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.USU152JIdu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H7sp1J9yyF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.USU152JIdu
++ cat /tmp/tmp.H7sp1J9yyF
++ rm /tmp/tmp.USU152JIdu /tmp/tmp.H7sp1J9yyF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 20
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.LbCnmMWf03/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pvc-auto-resize-18540", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PpJLKSzPS4
++ mktemp
+ local LAST_ERR=/tmp/tmp.WbloxKryfa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PpJLKSzPS4
+ cat /tmp/tmp.WbloxKryfa
+ rm /tmp/tmp.PpJLKSzPS4 /tmp/tmp.WbloxKryfa
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.LbCnmMWf03/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.LbCnmMWf03/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.LbCnmMWf03/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml /tmp/tmp.LbCnmMWf03/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-02-03T01:51:35+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-18540
+ local 'command=db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-18540
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-18540 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fy38uXmcRl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Uhq8zt0BBp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fy38uXmcRl
++ cat /tmp/tmp.Uhq8zt0BBp
++ rm /tmp/tmp.fy38uXmcRl /tmp/tmp.Uhq8zt0BBp
++ return 0
+ local client_container=psmdb-client-86cb5d8484-qqcws
+ kubectl_bin exec psmdb-client-86cb5d8484-qqcws -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-18540.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rhlnVJKjEY
++ mktemp
+ local LAST_ERR=/tmp/tmp.bCVj2ROalf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-86cb5d8484-qqcws -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-18540.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rhlnVJKjEY
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("ce1eb801-c5b6-4c9e-9ceb-5e0853513928") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.bCVj2ROalf
+ rm /tmp/tmp.rhlnVJKjEY /tmp/tmp.bCVj2ROalf
+ return 0
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pvc-auto-resize-18540
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pvc-auto-resize-18540
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.pvc-auto-resize-18540 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yScPniALqO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HCF6YDIo3D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yScPniALqO
++ cat /tmp/tmp.HCF6YDIo3D
++ rm /tmp/tmp.yScPniALqO /tmp/tmp.HCF6YDIo3D
++ return 0
+ local client_container=psmdb-client-86cb5d8484-qqcws
+ kubectl_bin exec psmdb-client-86cb5d8484-qqcws -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pvc-auto-resize-18540.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WsHSOycw5O
++ mktemp
+ local LAST_ERR=/tmp/tmp.7MWZro0WgY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-86cb5d8484-qqcws -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pvc-auto-resize-18540.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WsHSOycw5O
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pvc-auto-resize-18540.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("06e07ded-736c-466a-97f6-de3dfc0f0011") }
Percona Server for MongoDB server version: v8.0.17-6
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.7MWZro0WgY
+ rm /tmp/tmp.WsHSOycw5O /tmp/tmp.7MWZro0WgY
+ return 0
+ desc 'verify initial PVC size is 1Gi'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify initial PVC size is 1Gi
-----------------------------------------------------------------------------------

++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MKxL7U8MdF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.duSpY5mIn1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MKxL7U8MdF
++ cat /tmp/tmp.duSpY5mIn1
++ rm /tmp/tmp.MKxL7U8MdF /tmp/tmp.duSpY5mIn1
++ return 0
+ initial_size=1Gi
+ echo 'Initial PVC size: 1Gi'
Initial PVC size: 1Gi
+ [[ 1Gi != \1\G\i ]]
+ desc 'fill disk to trigger autoscaling threshold 80 percent'
+ set +o xtrace


-----------------------------------------------------------------------------------
fill disk to trigger autoscaling threshold 80 percent
-----------------------------------------------------------------------------------

+ fill_disk_to_threshold some-name-rs0-0 80 850
+ local pod=some-name-rs0-0
+ local threshold=80
+ local size_mb=850
+ echo 'Filling disk on some-name-rs0-0 to approximately 80% (creating 850MB file)'
Filling disk on some-name-rs0-0 to approximately 80% (creating 850MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=850 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.I8TZ8F8qAA
++ mktemp
+ local LAST_ERR=/tmp/tmp.SZiJQ56ZXP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=850 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.I8TZ8F8qAA
+ cat /tmp/tmp.SZiJQ56ZXP
+ rm /tmp/tmp.I8TZ8F8qAA /tmp/tmp.SZiJQ56ZXP
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1UEFTFA6Nb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TUhgdNnDGN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1UEFTFA6Nb
++ cat /tmp/tmp.TUhgdNnDGN
++ rm /tmp/tmp.1UEFTFA6Nb /tmp/tmp.TUhgdNnDGN
++ return 0
+ usage=100
+ echo 'Current disk usage on some-name-rs0-0: 100%'
Current disk usage on some-name-rs0-0: 100%
+ desc 'wait for auto-resize to trigger'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for auto-resize to trigger
-----------------------------------------------------------------------------------

+ wait_for_auto_resize mongod-data-some-name-rs0-0 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F18ynXbaBC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LwBhZ92XNy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F18ynXbaBC
++ cat /tmp/tmp.LwBhZ92XNy
++ rm /tmp/tmp.F18ynXbaBC /tmp/tmp.LwBhZ92XNy
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ecixRLghuj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pq5yaQYjsI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ecixRLghuj
++ cat /tmp/tmp.Pq5yaQYjsI
++ rm /tmp/tmp.ecixRLghuj /tmp/tmp.Pq5yaQYjsI
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 3Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BZi2zgI1iU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AB6DJKy3RJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BZi2zgI1iU
++ cat /tmp/tmp.AB6DJKy3RJ
++ rm /tmp/tmp.BZi2zgI1iU /tmp/tmp.AB6DJKy3RJ
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FVN8GBbGHL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lV4iMk0R8u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FVN8GBbGHL
++ cat /tmp/tmp.lV4iMk0R8u
++ rm /tmp/tmp.FVN8GBbGHL /tmp/tmp.lV4iMk0R8u
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LWO7rO95bE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZlcjNcrt4p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LWO7rO95bE
++ cat /tmp/tmp.ZlcjNcrt4p
++ rm /tmp/tmp.LWO7rO95bE /tmp/tmp.ZlcjNcrt4p
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VTAB1AgRFg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vYKv3AIxd1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VTAB1AgRFg
++ cat /tmp/tmp.vYKv3AIxd1
++ rm /tmp/tmp.VTAB1AgRFg /tmp/tmp.vYKv3AIxd1
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.O93nlH4P45
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YUGbVYgwmB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.O93nlH4P45
++ cat /tmp/tmp.YUGbVYgwmB
++ rm /tmp/tmp.O93nlH4P45 /tmp/tmp.YUGbVYgwmB
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 4 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=5
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wmtgkWmB8W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zFmKjTcf8e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wmtgkWmB8W
++ cat /tmp/tmp.zFmKjTcf8e
++ rm /tmp/tmp.wmtgkWmB8W /tmp/tmp.zFmKjTcf8e
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 5 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=6
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oUkRfWawsA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UW3F0rSy8e
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oUkRfWawsA
++ cat /tmp/tmp.UW3F0rSy8e
++ rm /tmp/tmp.oUkRfWawsA /tmp/tmp.UW3F0rSy8e
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 3Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OezKrVVAyQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WRW8y9DnkD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OezKrVVAyQ
++ cat /tmp/tmp.WRW8y9DnkD
++ rm /tmp/tmp.OezKrVVAyQ /tmp/tmp.WRW8y9DnkD
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 3Gi
+ desc 'verify autoscaling status is updated'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify autoscaling status is updated
-----------------------------------------------------------------------------------

+ check_autoscaling_status some-name 1 mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ local cluster=some-name
+ local expected_resize_count=1
+ shift 2
+ pvc_names=('mongod-data-some-name-rs0-0' 'mongod-data-some-name-rs0-1' 'mongod-data-some-name-rs0-2')
+ local pvc_names
+ local max_retry=30
+ local sleep_time=10
+ echo 'Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2'
Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ echo 'Expected resize count per PVC: 1'
Expected resize count per PVC: 1
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qPpKa5tgTV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xGDZoYkJCQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qPpKa5tgTV
++ cat /tmp/tmp.xGDZoYkJCQ
++ rm /tmp/tmp.qPpKa5tgTV /tmp/tmp.xGDZoYkJCQ
++ return 0
+ [[ '' -eq 1 ]]
+ [[ 0 -ge 30 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZCMz7IlVVL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BXaqmsWRlQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZCMz7IlVVL
++ cat /tmp/tmp.BXaqmsWRlQ
++ rm /tmp/tmp.ZCMz7IlVVL /tmp/tmp.BXaqmsWRlQ
++ return 0
+ [[ 1 -eq 1 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KHjLgIJzfl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i7K6dDkIxp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KHjLgIJzfl
++ cat /tmp/tmp.i7K6dDkIxp
++ rm /tmp/tmp.KHjLgIJzfl /tmp/tmp.i7K6dDkIxp
++ return 0
+ [[ 1 -eq 1 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CgNoPBhgYk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yJzVUjHhi1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CgNoPBhgYk
++ cat /tmp/tmp.yJzVUjHhi1
++ rm /tmp/tmp.CgNoPBhgYk /tmp/tmp.yJzVUjHhi1
++ return 0
+ [[ 1 -eq 1 ]]
+ desc 'wait for cluster consistency after resize'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for cluster consistency after resize
-----------------------------------------------------------------------------------

+ wait_cluster_consistency some-name 64
+ local cluster_name=some-name
+ local wait_time=64
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5snLupIz3k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EGEIUXomn7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5snLupIz3k
++ cat /tmp/tmp.EGEIUXomn7
++ rm /tmp/tmp.5snLupIz3k /tmp/tmp.EGEIUXomn7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ desc 'verify cluster is ready'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify cluster is ready
-----------------------------------------------------------------------------------

+ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+ grep -q ready
++ mktemp
+ local LAST_OUT=/tmp/tmp.VO1SCIApUL
++ mktemp
+ local LAST_ERR=/tmp/tmp.5awdyZ0mqe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VO1SCIApUL
+ cat /tmp/tmp.5awdyZ0mqe
+ rm /tmp/tmp.VO1SCIApUL /tmp/tmp.5awdyZ0mqe
+ return 0
+ desc 'test namespace resource quota enforcement'
+ set +o xtrace


-----------------------------------------------------------------------------------
test namespace resource quota enforcement
-----------------------------------------------------------------------------------

+ apply_resourcequota 9Gi
+ local quota=9Gi
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4vBYO65nA0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GnfBHTQsJt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4vBYO65nA0
++ cat /tmp/tmp.GnfBHTQsJt
++ rm /tmp/tmp.4vBYO65nA0 /tmp/tmp.GnfBHTQsJt
++ return 0
+ default_sc=standard-rwo
+ echo 'Applying resourcequota for default storageclass standard-rwo with quota 9Gi'
Applying resourcequota for default storageclass standard-rwo with quota 9Gi
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/resourcequota.yml
+ sed s/STORAGECLASS/standard-rwo/
+ sed s/QUOTA/9Gi/
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Nha9NYoO5A
++ mktemp
+ local LAST_ERR=/tmp/tmp.S58Y9tC3sH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Nha9NYoO5A
resourcequota/default-storage-quota created
+ cat /tmp/tmp.S58Y9tC3sH
+ rm /tmp/tmp.Nha9NYoO5A /tmp/tmp.S58Y9tC3sH
+ return 0
+ echo 'Filling disk again to trigger autoscaling with quota limit'
Filling disk again to trigger autoscaling with quota limit
+ fill_disk_to_threshold some-name-rs0-0 80 2400
+ local pod=some-name-rs0-0
+ local threshold=80
+ local size_mb=2400
+ echo 'Filling disk on some-name-rs0-0 to approximately 80% (creating 2400MB file)'
Filling disk on some-name-rs0-0 to approximately 80% (creating 2400MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=2400 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.dDvirF0Su2
++ mktemp
+ local LAST_ERR=/tmp/tmp.ABC8xXVZqK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=2400 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dDvirF0Su2
+ cat /tmp/tmp.ABC8xXVZqK
+ rm /tmp/tmp.dDvirF0Su2 /tmp/tmp.ABC8xXVZqK
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gUnyuOToYm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EVgUGwGelg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gUnyuOToYm
++ cat /tmp/tmp.EVgUGwGelg
++ rm /tmp/tmp.gUnyuOToYm /tmp/tmp.EVgUGwGelg
++ return 0
+ usage=88
+ echo 'Current disk usage on some-name-rs0-0: 88%'
Current disk usage on some-name-rs0-0: 88%
+ echo 'Waiting to verify PVC does not resize beyond quota (should stay at 3Gi)'
Waiting to verify PVC does not resize beyond quota (should stay at 3Gi)
+ sleep 30
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p7iiS23qgJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GCOIjMyHEl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.p7iiS23qgJ
++ cat /tmp/tmp.GCOIjMyHEl
++ rm /tmp/tmp.p7iiS23qgJ /tmp/tmp.GCOIjMyHEl
++ return 0
+ current_size=3Gi
+ [[ 3Gi != \3\G\i ]]
+ desc 'increase quota to allow further autoscaling'
+ set +o xtrace


-----------------------------------------------------------------------------------
increase quota to allow further autoscaling
-----------------------------------------------------------------------------------

+ apply_resourcequota 15Gi
+ local quota=15Gi
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qPAZZxq1nS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8eWt3stpJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qPAZZxq1nS
++ cat /tmp/tmp.8eWt3stpJ0
++ rm /tmp/tmp.qPAZZxq1nS /tmp/tmp.8eWt3stpJ0
++ return 0
+ default_sc=standard-rwo
+ echo 'Applying resourcequota for default storageclass standard-rwo with quota 15Gi'
Applying resourcequota for default storageclass standard-rwo with quota 15Gi
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2227/e2e-tests/pvc-auto-resize/conf/resourcequota.yml
+ sed s/STORAGECLASS/standard-rwo/
+ sed s/QUOTA/15Gi/
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8MGxWiki8x
++ mktemp
+ local LAST_ERR=/tmp/tmp.MVHeo6eZWX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8MGxWiki8x
resourcequota/default-storage-quota configured
+ cat /tmp/tmp.MVHeo6eZWX
+ rm /tmp/tmp.8MGxWiki8x /tmp/tmp.MVHeo6eZWX
+ return 0
+ wait_for_auto_resize mongod-data-some-name-rs0-0 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BkDkmXOcsJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B61UyBonaR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BkDkmXOcsJ
++ cat /tmp/tmp.B61UyBonaR
++ rm /tmp/tmp.BkDkmXOcsJ /tmp/tmp.B61UyBonaR
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YqkSZS3dEz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nJjZdcmfeD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YqkSZS3dEz
++ cat /tmp/tmp.nJjZdcmfeD
++ rm /tmp/tmp.YqkSZS3dEz /tmp/tmp.nJjZdcmfeD
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 5Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9mSbOh08ps
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zxgyQA7Evw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9mSbOh08ps
++ cat /tmp/tmp.zxgyQA7Evw
++ rm /tmp/tmp.9mSbOh08ps /tmp/tmp.zxgyQA7Evw
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1Z5st6ZXPQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ga0CgYuiyO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1Z5st6ZXPQ
++ cat /tmp/tmp.ga0CgYuiyO
++ rm /tmp/tmp.1Z5st6ZXPQ /tmp/tmp.ga0CgYuiyO
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MOmbaxEy9I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AFrYSDUdmQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MOmbaxEy9I
++ cat /tmp/tmp.AFrYSDUdmQ
++ rm /tmp/tmp.MOmbaxEy9I /tmp/tmp.AFrYSDUdmQ
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZQvjMVppBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ma0nrGj1fT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZQvjMVppBI
++ cat /tmp/tmp.Ma0nrGj1fT
++ rm /tmp/tmp.ZQvjMVppBI /tmp/tmp.Ma0nrGj1fT
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nmSoRncV9n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RiKFSBvAPL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nmSoRncV9n
++ cat /tmp/tmp.RiKFSBvAPL
++ rm /tmp/tmp.nmSoRncV9n /tmp/tmp.RiKFSBvAPL
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 4 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=5
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HAjSJNhqIs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A12ttdueo1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HAjSJNhqIs
++ cat /tmp/tmp.A12ttdueo1
++ rm /tmp/tmp.HAjSJNhqIs /tmp/tmp.A12ttdueo1
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 5 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=6
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Z81nyL6yt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dug0YEC85d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9Z81nyL6yt
++ cat /tmp/tmp.Dug0YEC85d
++ rm /tmp/tmp.9Z81nyL6yt /tmp/tmp.Dug0YEC85d
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 5Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vfa8gvWLYr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ebgp8DYlFz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vfa8gvWLYr
++ cat /tmp/tmp.ebgp8DYlFz
++ rm /tmp/tmp.vfa8gvWLYr /tmp/tmp.ebgp8DYlFz
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 5Gi
+ desc 'verify cluster consistency after quota test'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify cluster consistency after quota test
-----------------------------------------------------------------------------------

+ wait_cluster_consistency some-name 64
+ local cluster_name=some-name
+ local wait_time=64
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XbYx4loOCm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nUck2WwQiw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XbYx4loOCm
++ cat /tmp/tmp.nUck2WwQiw
++ rm /tmp/tmp.XbYx4loOCm /tmp/tmp.nUck2WwQiw
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kAQcBilOpw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3R105OvTAp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kAQcBilOpw
++ cat /tmp/tmp.3R105OvTAp
++ rm /tmp/tmp.kAQcBilOpw /tmp/tmp.3R105OvTAp
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eQ6Xmw5WOX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ed5bs504qs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eQ6Xmw5WOX
++ cat /tmp/tmp.Ed5bs504qs
++ rm /tmp/tmp.eQ6Xmw5WOX /tmp/tmp.Ed5bs504qs
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5z4AiqaQVV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gV24AeAjpZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5z4AiqaQVV
++ cat /tmp/tmp.gV24AeAjpZ
++ rm /tmp/tmp.5z4AiqaQVV /tmp/tmp.gV24AeAjpZ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.g4Q3WcUZRb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bm4glDaMEJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.g4Q3WcUZRb
++ cat /tmp/tmp.bm4glDaMEJ
++ rm /tmp/tmp.g4Q3WcUZRb /tmp/tmp.bm4glDaMEJ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jjSBNqPouT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.auS1LlTIVF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jjSBNqPouT
++ cat /tmp/tmp.auS1LlTIVF
++ rm /tmp/tmp.jjSBNqPouT /tmp/tmp.auS1LlTIVF
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4JRzQ6d6cl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qASb1xIC8a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4JRzQ6d6cl
++ cat /tmp/tmp.qASb1xIC8a
++ rm /tmp/tmp.4JRzQ6d6cl /tmp/tmp.qASb1xIC8a
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZawdLYvik7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ORYUvSTYET
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZawdLYvik7
++ cat /tmp/tmp.ORYUvSTYET
++ rm /tmp/tmp.ZawdLYvik7 /tmp/tmp.ORYUvSTYET
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z8e2QE30w7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2RWEckS2up
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z8e2QE30w7
++ cat /tmp/tmp.2RWEckS2up
++ rm /tmp/tmp.Z8e2QE30w7 /tmp/tmp.2RWEckS2up
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fq5UWcLRq7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9CxgEGjM4s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fq5UWcLRq7
++ cat /tmp/tmp.9CxgEGjM4s
++ rm /tmp/tmp.fq5UWcLRq7 /tmp/tmp.9CxgEGjM4s
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KXIA55CYVJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W3vCU0cTq1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KXIA55CYVJ
++ cat /tmp/tmp.W3vCU0cTq1
++ rm /tmp/tmp.KXIA55CYVJ /tmp/tmp.W3vCU0cTq1
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uzV0xeF5wd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pspt3IuNX3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uzV0xeF5wd
++ cat /tmp/tmp.pspt3IuNX3
++ rm /tmp/tmp.uzV0xeF5wd /tmp/tmp.pspt3IuNX3
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RSLRGBINQu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rhTr0rruMG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RSLRGBINQu
++ cat /tmp/tmp.rhTr0rruMG
++ rm /tmp/tmp.RSLRGBINQu /tmp/tmp.rhTr0rruMG
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4vzqRrEtcA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Oy1WCIb7WB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4vzqRrEtcA
++ cat /tmp/tmp.Oy1WCIb7WB
++ rm /tmp/tmp.4vzqRrEtcA /tmp/tmp.Oy1WCIb7WB
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IN4QrYFkiC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rVUIhjXY3D
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IN4QrYFkiC
++ cat /tmp/tmp.rVUIhjXY3D
++ rm /tmp/tmp.IN4QrYFkiC /tmp/tmp.rVUIhjXY3D
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nWMQZyMVUy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0jvjt6HDHZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nWMQZyMVUy
++ cat /tmp/tmp.0jvjt6HDHZ
++ rm /tmp/tmp.nWMQZyMVUy /tmp/tmp.0jvjt6HDHZ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sEH1JF6Nlm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XECejKRqWA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sEH1JF6Nlm
++ cat /tmp/tmp.XECejKRqWA
++ rm /tmp/tmp.sEH1JF6Nlm /tmp/tmp.XECejKRqWA
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 17 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nZP3CnTkFU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.e5abUnXwHI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nZP3CnTkFU
++ cat /tmp/tmp.e5abUnXwHI
++ rm /tmp/tmp.nZP3CnTkFU /tmp/tmp.e5abUnXwHI
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 18 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vGoGjfJ9nP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uEeh1OXR6w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vGoGjfJ9nP
++ cat /tmp/tmp.uEeh1OXR6w
++ rm /tmp/tmp.vGoGjfJ9nP /tmp/tmp.uEeh1OXR6w
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 19 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f8QfZmIgVi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m61OQhhGsN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f8QfZmIgVi
++ cat /tmp/tmp.m61OQhhGsN
++ rm /tmp/tmp.f8QfZmIgVi /tmp/tmp.m61OQhhGsN
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 20 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SWd8FaN77E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZR1UjKvYrH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SWd8FaN77E
++ cat /tmp/tmp.ZR1UjKvYrH
++ rm /tmp/tmp.SWd8FaN77E /tmp/tmp.ZR1UjKvYrH
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 21 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DbtgnudYWT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HKbPQ1xyPH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DbtgnudYWT
++ cat /tmp/tmp.HKbPQ1xyPH
++ rm /tmp/tmp.DbtgnudYWT /tmp/tmp.HKbPQ1xyPH
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 22 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PTcwjffA2s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.41VfAO4DB2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PTcwjffA2s
++ cat /tmp/tmp.41VfAO4DB2
++ rm /tmp/tmp.PTcwjffA2s /tmp/tmp.41VfAO4DB2
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 23 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iMUi9HFC6j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t0ct2JrEID
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iMUi9HFC6j
++ cat /tmp/tmp.t0ct2JrEID
++ rm /tmp/tmp.iMUi9HFC6j /tmp/tmp.t0ct2JrEID
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 24 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dsQBanEm4x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B6Sggqtca2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dsQBanEm4x
++ cat /tmp/tmp.B6Sggqtca2
++ rm /tmp/tmp.dsQBanEm4x /tmp/tmp.B6Sggqtca2
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 25 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kcly7wrRZm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jWUVB2OPB5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kcly7wrRZm
++ cat /tmp/tmp.jWUVB2OPB5
++ rm /tmp/tmp.kcly7wrRZm /tmp/tmp.jWUVB2OPB5
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 26 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X4nRPdbQQE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pI6LFB2J2c
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X4nRPdbQQE
++ cat /tmp/tmp.pI6LFB2J2c
++ rm /tmp/tmp.X4nRPdbQQE /tmp/tmp.pI6LFB2J2c
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 27 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QwpVaFatwy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oawcpvbppK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QwpVaFatwy
++ cat /tmp/tmp.oawcpvbppK
++ rm /tmp/tmp.QwpVaFatwy /tmp/tmp.oawcpvbppK
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 28 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.llPhNFID2A
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XvcQrLDfMD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.llPhNFID2A
++ cat /tmp/tmp.XvcQrLDfMD
++ rm /tmp/tmp.llPhNFID2A /tmp/tmp.XvcQrLDfMD
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 29 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4jgl6YHXwz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BQjIujXqCm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4jgl6YHXwz
++ cat /tmp/tmp.BQjIujXqCm
++ rm /tmp/tmp.4jgl6YHXwz /tmp/tmp.BQjIujXqCm
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 30 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vVdZuUFW7m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OeH4RPFMM6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vVdZuUFW7m
++ cat /tmp/tmp.OeH4RPFMM6
++ rm /tmp/tmp.vVdZuUFW7m /tmp/tmp.OeH4RPFMM6
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 31 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vhOycBAVjR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5hBS0wRW7n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vhOycBAVjR
++ cat /tmp/tmp.5hBS0wRW7n
++ rm /tmp/tmp.vhOycBAVjR /tmp/tmp.5hBS0wRW7n
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 32 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.keNiiEm0o8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CUALWbQ26B
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.keNiiEm0o8
++ cat /tmp/tmp.CUALWbQ26B
++ rm /tmp/tmp.keNiiEm0o8 /tmp/tmp.CUALWbQ26B
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 33 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GTuHUsAdcN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NTjUTe7Udr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GTuHUsAdcN
++ cat /tmp/tmp.NTjUTe7Udr
++ rm /tmp/tmp.GTuHUsAdcN /tmp/tmp.NTjUTe7Udr
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 34 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u4ASNzkYof
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aV0D64mVIM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u4ASNzkYof
++ cat /tmp/tmp.aV0D64mVIM
++ rm /tmp/tmp.u4ASNzkYof /tmp/tmp.aV0D64mVIM
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 35 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F2aglWP8Uf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DI9SYzEIHS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F2aglWP8Uf
++ cat /tmp/tmp.DI9SYzEIHS
++ rm /tmp/tmp.F2aglWP8Uf /tmp/tmp.DI9SYzEIHS
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 36 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bPclIGm9y9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.shU3CDI3AD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bPclIGm9y9
++ cat /tmp/tmp.shU3CDI3AD
++ rm /tmp/tmp.bPclIGm9y9 /tmp/tmp.shU3CDI3AD
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 37 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DXqy8s5Cmk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7u8EzwT1hG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DXqy8s5Cmk
++ cat /tmp/tmp.7u8EzwT1hG
++ rm /tmp/tmp.DXqy8s5Cmk /tmp/tmp.7u8EzwT1hG
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 38 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S8yTxCdCAy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E6kUgavh7F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S8yTxCdCAy
++ cat /tmp/tmp.E6kUgavh7F
++ rm /tmp/tmp.S8yTxCdCAy /tmp/tmp.E6kUgavh7F
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 39 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aqzy0bE5kg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lVdlbpualM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aqzy0bE5kg
++ cat /tmp/tmp.lVdlbpualM
++ rm /tmp/tmp.aqzy0bE5kg /tmp/tmp.lVdlbpualM
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 40 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FemAnJytmF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JOz8vf1p5l
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FemAnJytmF
++ cat /tmp/tmp.JOz8vf1p5l
++ rm /tmp/tmp.FemAnJytmF /tmp/tmp.JOz8vf1p5l
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 41 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UzhufndebF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ez28goqgjo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UzhufndebF
++ cat /tmp/tmp.Ez28goqgjo
++ rm /tmp/tmp.UzhufndebF /tmp/tmp.Ez28goqgjo
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 42 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IGEBGupegs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wR0HCBsW9K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IGEBGupegs
++ cat /tmp/tmp.wR0HCBsW9K
++ rm /tmp/tmp.IGEBGupegs /tmp/tmp.wR0HCBsW9K
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 43 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U25jdUfJOa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kf2qszHKbz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.U25jdUfJOa
++ cat /tmp/tmp.kf2qszHKbz
++ rm /tmp/tmp.U25jdUfJOa /tmp/tmp.kf2qszHKbz
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 44 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FI4ZMDspZb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jqwRNeZ9AC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FI4ZMDspZb
++ cat /tmp/tmp.jqwRNeZ9AC
++ rm /tmp/tmp.FI4ZMDspZb /tmp/tmp.jqwRNeZ9AC
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 45 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D3For63jTo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MvrHD8ttBh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.D3For63jTo
++ cat /tmp/tmp.MvrHD8ttBh
++ rm /tmp/tmp.D3For63jTo /tmp/tmp.MvrHD8ttBh
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 46 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ji6A1dXHw1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LPJ44Q1jHF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ji6A1dXHw1
++ cat /tmp/tmp.LPJ44Q1jHF
++ rm /tmp/tmp.Ji6A1dXHw1 /tmp/tmp.LPJ44Q1jHF
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 47 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PcsyBYRPcj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Hs5TELS8jd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PcsyBYRPcj
++ cat /tmp/tmp.Hs5TELS8jd
++ rm /tmp/tmp.PcsyBYRPcj /tmp/tmp.Hs5TELS8jd
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 48 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ydBAFSQDE1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tkupIMiDgU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ydBAFSQDE1
++ cat /tmp/tmp.tkupIMiDgU
++ rm /tmp/tmp.ydBAFSQDE1 /tmp/tmp.tkupIMiDgU
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 49 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KUJd1PEJxQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d5sx2djTCu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KUJd1PEJxQ
++ cat /tmp/tmp.d5sx2djTCu
++ rm /tmp/tmp.KUJd1PEJxQ /tmp/tmp.d5sx2djTCu
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 50 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sETHHgBoip
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S8tCF5vftQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sETHHgBoip
++ cat /tmp/tmp.S8tCF5vftQ
++ rm /tmp/tmp.sETHHgBoip /tmp/tmp.S8tCF5vftQ
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 51 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BUsZJ6uod3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZNqAEJVVNN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BUsZJ6uod3
++ cat /tmp/tmp.ZNqAEJVVNN
++ rm /tmp/tmp.BUsZJ6uod3 /tmp/tmp.ZNqAEJVVNN
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 52 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0gJEMlqIbE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7pDRQuJX9p
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0gJEMlqIbE
++ cat /tmp/tmp.7pDRQuJX9p
++ rm /tmp/tmp.0gJEMlqIbE /tmp/tmp.7pDRQuJX9p
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 53 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z4sLP37z3P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QPvCia2egc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z4sLP37z3P
++ cat /tmp/tmp.QPvCia2egc
++ rm /tmp/tmp.Z4sLP37z3P /tmp/tmp.QPvCia2egc
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 54 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HBGRnag9aK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K2KNiFCaCT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HBGRnag9aK
++ cat /tmp/tmp.K2KNiFCaCT
++ rm /tmp/tmp.HBGRnag9aK /tmp/tmp.K2KNiFCaCT
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 55 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OL7ZChSs8h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wwJpMlko3J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OL7ZChSs8h
++ cat /tmp/tmp.wwJpMlko3J
++ rm /tmp/tmp.OL7ZChSs8h /tmp/tmp.wwJpMlko3J
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 56 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.marV51OJPy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MvSzYtVJVL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.marV51OJPy
++ cat /tmp/tmp.MvSzYtVJVL
++ rm /tmp/tmp.marV51OJPy /tmp/tmp.MvSzYtVJVL
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 57 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ELKs8DNwlG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q8OxNzZgeR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ELKs8DNwlG
++ cat /tmp/tmp.q8OxNzZgeR
++ rm /tmp/tmp.ELKs8DNwlG /tmp/tmp.q8OxNzZgeR
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 58 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6zROA9tun9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nR4sdSCm1w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6zROA9tun9
++ cat /tmp/tmp.nR4sdSCm1w
++ rm /tmp/tmp.6zROA9tun9 /tmp/tmp.nR4sdSCm1w
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 59 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LcnHsIdGod
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PsoSsqnUDA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LcnHsIdGod
++ cat /tmp/tmp.PsoSsqnUDA
++ rm /tmp/tmp.LcnHsIdGod /tmp/tmp.PsoSsqnUDA
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 60 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DZFBGrtWmp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TNpOx4XT4I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DZFBGrtWmp
++ cat /tmp/tmp.TNpOx4XT4I
++ rm /tmp/tmp.DZFBGrtWmp /tmp/tmp.TNpOx4XT4I
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 61 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EUDeUayX3g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HsARi5M0G2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EUDeUayX3g
++ cat /tmp/tmp.HsARi5M0G2
++ rm /tmp/tmp.EUDeUayX3g /tmp/tmp.HsARi5M0G2
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 62 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vBM772EvUc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gnp0NtEGeI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vBM772EvUc
++ cat /tmp/tmp.gnp0NtEGeI
++ rm /tmp/tmp.vBM772EvUc /tmp/tmp.gnp0NtEGeI
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 63 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0JPR25p9XR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HErStkzN1t
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0JPR25p9XR
++ cat /tmp/tmp.HErStkzN1t
++ rm /tmp/tmp.0JPR25p9XR /tmp/tmp.HErStkzN1t
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 64 -ge 64 ']'
+ echo max retry count 64 reached. something went wrong with operator or kubernetes cluster
max retry count 64 reached. something went wrong with operator or kubernetes cluster
+ exit 1