Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/logs/pvc-auto-resize.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ '[' 0 == 1 ']'
+ ensure_default_sc_allows_expansion
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.97ir5hAz6W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3sKwci9brr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.97ir5hAz6W
++ cat /tmp/tmp.3sKwci9brr
++ rm /tmp/tmp.97ir5hAz6W /tmp/tmp.3sKwci9brr
++ return 0
+ default_sc=standard-rwo
+ echo 'Checking if default storageclass standard-rwo allows volume expansion'
Checking if default storageclass standard-rwo allows volume expansion
+ local allowVolumeExpansion
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.name=="standard-rwo")].allowVolumeExpansion}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t4Ye9L9AUY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4q9AP7RNVI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.name=="standard-rwo")].allowVolumeExpansion}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t4Ye9L9AUY
++ cat /tmp/tmp.4q9AP7RNVI
++ rm /tmp/tmp.t4Ye9L9AUY /tmp/tmp.4q9AP7RNVI
++ return 0
+ allowVolumeExpansion=true
+ [[ true != \t\r\u\e ]]
+ create_infra pvc-auto-resize-23145
+ local ns=pvc-auto-resize-23145
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.8lZssVP0TQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.k2vPteN2uz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8lZssVP0TQ
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.k2vPteN2uz
+ rm /tmp/tmp.8lZssVP0TQ /tmp/tmp.k2vPteN2uz
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.WsbFeBo0Xu
++ mktemp
+ local LAST_ERR=/tmp/tmp.wNAmBB2Jlu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WsbFeBo0Xu
+ cat /tmp/tmp.wNAmBB2Jlu
+ rm /tmp/tmp.WsbFeBo0Xu /tmp/tmp.wNAmBB2Jlu
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.cmYQwIEQA2
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bv4Cpiqavm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cmYQwIEQA2
+ cat /tmp/tmp.Bv4Cpiqavm
+ rm /tmp/tmp.cmYQwIEQA2 /tmp/tmp.Bv4Cpiqavm
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.YHhtvqsxdj
++ mktemp
+ local LAST_ERR=/tmp/tmp.LOqCUTfVIk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YHhtvqsxdj
+ cat /tmp/tmp.LOqCUTfVIk
+ rm /tmp/tmp.YHhtvqsxdj /tmp/tmp.LOqCUTfVIk
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RQ6c9TVCAn
++ mktemp
+ local LAST_ERR=/tmp/tmp.ENcZftv1NL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RQ6c9TVCAn
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.ENcZftv1NL
+ rm /tmp/tmp.RQ6c9TVCAn /tmp/tmp.ENcZftv1NL
+ return 0
+ check_crd_for_deletion PR-2270-9169a6cb
+ local git_tag=PR-2270-9169a6cb
++ yq eval .metadata.name
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2270-9169a6cb/deploy/crd.yaml
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/sbin/sed s/---//g
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w60xA8iXKy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jxfpfz7Ugk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.w60xA8iXKy
++ cat /tmp/tmp.jxfpfz7Ugk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.w60xA8iXKy
++ cat /tmp/tmp.jxfpfz7Ugk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.w60xA8iXKy
++ cat /tmp/tmp.jxfpfz7Ugk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.w60xA8iXKy
++ cat /tmp/tmp.jxfpfz7Ugk
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.w60xA8iXKy /tmp/tmp.jxfpfz7Ugk
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.s15OKb8j06
+ local LAST_OUT=/tmp/tmp.ub1njGE4mN
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.duaEFknCrQ
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.Hcq9VqxdzV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s15OKb8j06
+ cat /tmp/tmp.duaEFknCrQ
+ rm /tmp/tmp.s15OKb8j06 /tmp/tmp.duaEFknCrQ
+ return 0
namespace "pvc-auto-resize-3364" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ub1njGE4mN
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.Hcq9VqxdzV
+ rm /tmp/tmp.ub1njGE4mN /tmp/tmp.Hcq9VqxdzV
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MOTQsJKb2Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.wK1L0jbUKY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MOTQsJKb2Y
+ cat /tmp/tmp.wK1L0jbUKY
+ rm /tmp/tmp.MOTQsJKb2Y /tmp/tmp.wK1L0jbUKY
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZeOCtCfaZT
++ mktemp
+ local LAST_ERR=/tmp/tmp.UAGRPhMnHB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZeOCtCfaZT
namespace/psmdb-operator created
+ cat /tmp/tmp.UAGRPhMnHB
+ rm /tmp/tmp.ZeOCtCfaZT /tmp/tmp.UAGRPhMnHB
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.f6zO3HC5Rf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MOQkNWP0qy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.f6zO3HC5Rf
++ cat /tmp/tmp.MOQkNWP0qy
++ rm /tmp/tmp.f6zO3HC5Rf /tmp/tmp.MOQkNWP0qy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.aAt4UcstbS
++ mktemp
+ local LAST_ERR=/tmp/tmp.sowUG61siz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aAt4UcstbS
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5" modified.
+ cat /tmp/tmp.sowUG61siz
+ rm /tmp/tmp.aAt4UcstbS /tmp/tmp.sowUG61siz
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2270-9169a6cb'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2270-9169a6cb
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.6HLA8LfAaz
++ mktemp
+ local LAST_ERR=/tmp/tmp.6SLCDARqlf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6HLA8LfAaz
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.6SLCDARqlf
+ rm /tmp/tmp.6HLA8LfAaz /tmp/tmp.6SLCDARqlf
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2yMvrbuGRd
++ mktemp
+ local LAST_ERR=/tmp/tmp.OfiE7VDWUF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2yMvrbuGRd
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.OfiE7VDWUF
+ rm /tmp/tmp.2yMvrbuGRd /tmp/tmp.OfiE7VDWUF
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2270-9169a6cb") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.VIsQQwNtH6
++ mktemp
+ local LAST_ERR=/tmp/tmp.lXInNTT5O4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VIsQQwNtH6
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.lXInNTT5O4
+ rm /tmp/tmp.VIsQQwNtH6 /tmp/tmp.lXInNTT5O4
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w8J3EAWpmK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nahQzm5fWM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w8J3EAWpmK
++ cat /tmp/tmp.nahQzm5fWM
++ rm /tmp/tmp.w8J3EAWpmK /tmp/tmp.nahQzm5fWM
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-dfd77fbfc-mww5n
+ local pod=percona-server-mongodb-operator-dfd77fbfc-mww5n
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-dfd77fbfc-mww5n to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
++ get_operator_pod
+ grep 'Manager starting up'
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EgTcNwDzvT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xzNAGp2NKQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EgTcNwDzvT
++ cat /tmp/tmp.xzNAGp2NKQ
++ rm /tmp/tmp.EgTcNwDzvT /tmp/tmp.xzNAGp2NKQ
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-dfd77fbfc-mww5n
++ mktemp
+ local LAST_OUT=/tmp/tmp.AOLKhGRHq1
++ mktemp
+ local LAST_ERR=/tmp/tmp.NQD34eHy2b
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-dfd77fbfc-mww5n
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AOLKhGRHq1
+ cat /tmp/tmp.NQD34eHy2b
+ rm /tmp/tmp.AOLKhGRHq1 /tmp/tmp.NQD34eHy2b
+ return 0
2026-03-26T09:50:03.994Z	INFO	setup	Manager starting up	{"gitCommit": "9169a6cb07cda16a30cfe6ec0278ae77b9077338", "gitBranch": "PR-2270-9169a6cb", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"}
+ create_namespace pvc-auto-resize-23145
+ local namespace=pvc-auto-resize-23145
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pvc-auto-resize-23145'
+ set +o xtrace
+ awk '{print$1}'


-----------------------------------------------------------------------------------
cleaned up old namespaces pvc-auto-resize-23145
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pvc-auto-resize-23145 --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ys9W41777c
++ mktemp
+ local LAST_ERR=/tmp/tmp.bYUU34givc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.qhx2eFwS5s
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.9T2Uq06nOw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace pvc-auto-resize-23145 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ys9W41777c
+ cat /tmp/tmp.bYUU34givc
+ rm /tmp/tmp.Ys9W41777c /tmp/tmp.bYUU34givc
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qhx2eFwS5s
+ cat /tmp/tmp.9T2Uq06nOw
+ rm /tmp/tmp.qhx2eFwS5s /tmp/tmp.9T2Uq06nOw
+ return 0
+ kubectl_bin wait --for=delete namespace pvc-auto-resize-23145
++ mktemp
+ local LAST_OUT=/tmp/tmp.XhCf8Tmrfm
++ mktemp
+ local LAST_ERR=/tmp/tmp.1zEZer1ZFa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace pvc-auto-resize-23145
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XhCf8Tmrfm
+ cat /tmp/tmp.1zEZer1ZFa
+ rm /tmp/tmp.XhCf8Tmrfm /tmp/tmp.1zEZer1ZFa
+ return 0
+ desc 'create namespace pvc-auto-resize-23145'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pvc-auto-resize-23145
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pvc-auto-resize-23145
++ mktemp
+ local LAST_OUT=/tmp/tmp.qKqQ2IhR1W
++ mktemp
+ local LAST_ERR=/tmp/tmp.YJ3GwBAinR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace pvc-auto-resize-23145
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qKqQ2IhR1W
namespace/pvc-auto-resize-23145 created
+ cat /tmp/tmp.YJ3GwBAinR
+ rm /tmp/tmp.qKqQ2IhR1W /tmp/tmp.YJ3GwBAinR
+ return 0
+ set_kube_ctx pvc-auto-resize-23145
+ local namespace=pvc-auto-resize-23145
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oRopGa6Okq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z4AEJ8h7Dg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oRopGa6Okq
++ cat /tmp/tmp.Z4AEJ8h7Dg
++ rm /tmp/tmp.oRopGa6Okq /tmp/tmp.Z4AEJ8h7Dg
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5 --namespace=pvc-auto-resize-23145
++ mktemp
+ local LAST_OUT=/tmp/tmp.tLbpwMo9l9
++ mktemp
+ local LAST_ERR=/tmp/tmp.g9Sgjf3qu4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5 --namespace=pvc-auto-resize-23145
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tLbpwMo9l9
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2270-9169a6cb-4-cluster5" modified.
+ cat /tmp/tmp.g9Sgjf3qu4
+ rm /tmp/tmp.tLbpwMo9l9 /tmp/tmp.g9Sgjf3qu4
+ return 0
+ desc 'create secrets and psmdb client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and psmdb client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.k2GrIAR8H9
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ad1p0Hlpq1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k2GrIAR8H9
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.Ad1p0Hlpq1
+ rm /tmp/tmp.k2GrIAR8H9 /tmp/tmp.Ad1p0Hlpq1
+ return 0
+ desc 'create PSMDB cluster with storageAutoscaling enabled'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PSMDB cluster with storageAutoscaling enabled
-----------------------------------------------------------------------------------

+ cluster=some-name
+ '[' 0 == 1 ']'
+ spinup_psmdb some-name-rs0 /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ local cluster=some-name-rs0
+ local config=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ local size=3
+ desc 'create first PSMDB cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2270-9169a6cb"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/pvc-auto-resize-23145/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.bNGO6tU7f4
++ mktemp
+ local LAST_ERR=/tmp/tmp.QCc66YNZ4G
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bNGO6tU7f4
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.QCc66YNZ4G
+ rm /tmp/tmp.bNGO6tU7f4 /tmp/tmp.QCc66YNZ4G
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready..............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LtMTsis8N4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FvodCBvNBe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LtMTsis8N4
++ cat /tmp/tmp.FvodCBvNBe
++ rm /tmp/tmp.LtMTsis8N4 /tmp/tmp.FvodCBvNBe
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.V5FO6avWv5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iwMetaXGIu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.V5FO6avWv5
++ cat /tmp/tmp.iwMetaXGIu
++ rm /tmp/tmp.V5FO6avWv5 /tmp/tmp.iwMetaXGIu
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LcVQ69pMgs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.geSs8XBATU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LcVQ69pMgs
++ cat /tmp/tmp.geSs8XBATU
++ rm /tmp/tmp.LcVQ69pMgs /tmp/tmp.geSs8XBATU
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 20
+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.LkB8c0ThLn/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("pvc-auto-resize-23145", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.XmrCDP6mdr
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZjwM7RkXUX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XmrCDP6mdr
+ cat /tmp/tmp.ZjwM7RkXUX
+ rm /tmp/tmp.XmrCDP6mdr /tmp/tmp.ZjwM7RkXUX
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.LkB8c0ThLn/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.32 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.LkB8c0ThLn/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.LkB8c0ThLn/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/compare/statefulset_some-name-rs0.yml /tmp/tmp.LkB8c0ThLn/statefulset_some-name-rs0.yml
+ log 'compare_kubectl: statefulset/some-name-rs0 OK'
+ set +o xtrace
[2026-03-26T09:52:44+0000] compare_kubectl: statefulset/some-name-rs0 OK
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-23145
+ local 'command=db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-23145
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-23145 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.16e70qFgKz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J5tAAvHWO5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.16e70qFgKz
++ cat /tmp/tmp.J5tAAvHWO5
++ rm /tmp/tmp.16e70qFgKz /tmp/tmp.J5tAAvHWO5
++ return 0
+ local client_container=psmdb-client-bb8b97679-wltvr
+ kubectl_bin exec psmdb-client-bb8b97679-wltvr -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-23145.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.j3PSZnhTPE
++ mktemp
+ local LAST_ERR=/tmp/tmp.RhcElwSvVp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wltvr -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.pvc-auto-resize-23145.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j3PSZnhTPE
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("82efce3b-a774-4367-a826-528ee14214bf") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.RhcElwSvVp
+ rm /tmp/tmp.j3PSZnhTPE /tmp/tmp.RhcElwSvVp
+ return 0
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.pvc-auto-resize-23145
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.pvc-auto-resize-23145
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@some-name-rs0.pvc-auto-resize-23145 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YT4L5Syqz7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ctotsxkQxo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YT4L5Syqz7
++ cat /tmp/tmp.ctotsxkQxo
++ rm /tmp/tmp.YT4L5Syqz7 /tmp/tmp.ctotsxkQxo
++ return 0
+ local client_container=psmdb-client-bb8b97679-wltvr
+ kubectl_bin exec psmdb-client-bb8b97679-wltvr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pvc-auto-resize-23145.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.igZ3f7j2tD
++ mktemp
+ local LAST_ERR=/tmp/tmp.9EKYZdGRbr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-bb8b97679-wltvr -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.pvc-auto-resize-23145.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.igZ3f7j2tD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.pvc-auto-resize-23145.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("05bcce74-4b0c-4f6b-9d32-ef8efe95e1aa") }
Percona Server for MongoDB server version: v8.0.19-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.9EKYZdGRbr
+ rm /tmp/tmp.igZ3f7j2tD /tmp/tmp.9EKYZdGRbr
+ return 0
+ desc 'verify initial PVC size is 1Gi'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify initial PVC size is 1Gi
-----------------------------------------------------------------------------------

++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fAmHy5XUsa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zuUNxYtFlm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fAmHy5XUsa
++ cat /tmp/tmp.zuUNxYtFlm
++ rm /tmp/tmp.fAmHy5XUsa /tmp/tmp.zuUNxYtFlm
++ return 0
+ initial_size=1Gi
+ echo 'Initial PVC size: 1Gi'
Initial PVC size: 1Gi
+ [[ 1Gi != \1\G\i ]]
+ desc 'fill disk to trigger autoscaling threshold 50 percent'
+ set +o xtrace


-----------------------------------------------------------------------------------
fill disk to trigger autoscaling threshold 50 percent
-----------------------------------------------------------------------------------

+ fill_disk_to_threshold some-name-rs0-0 50 600
+ local pod=some-name-rs0-0
+ local threshold=50
+ local size_mb=600
+ echo 'Filling disk on some-name-rs0-0 to approximately 50% (creating 600MB file)'
Filling disk on some-name-rs0-0 to approximately 50% (creating 600MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=600 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1HLisH90mb
++ mktemp
+ local LAST_ERR=/tmp/tmp.lgsXsSvolJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=600 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1HLisH90mb
+ cat /tmp/tmp.lgsXsSvolJ
+ rm /tmp/tmp.1HLisH90mb /tmp/tmp.lgsXsSvolJ
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XRz9oaDjmY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rIz0spIuJj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XRz9oaDjmY
++ cat /tmp/tmp.rIz0spIuJj
++ rm /tmp/tmp.XRz9oaDjmY /tmp/tmp.rIz0spIuJj
++ return 0
+ usage=85
+ echo 'Current disk usage on some-name-rs0-0: 85%'
Current disk usage on some-name-rs0-0: 85%
+ desc 'wait for auto-resize to trigger'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for auto-resize to trigger
-----------------------------------------------------------------------------------

+ wait_for_auto_resize mongod-data-some-name-rs0-0 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8YwRABHOvW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SiQmFMhTeP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8YwRABHOvW
++ cat /tmp/tmp.SiQmFMhTeP
++ rm /tmp/tmp.8YwRABHOvW /tmp/tmp.SiQmFMhTeP
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cg93t0tEmp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OzaaHTM5ce
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cg93t0tEmp
++ cat /tmp/tmp.OzaaHTM5ce
++ rm /tmp/tmp.cg93t0tEmp /tmp/tmp.OzaaHTM5ce
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.58C2xtHlYJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hVdiXnpKSy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.58C2xtHlYJ
++ cat /tmp/tmp.hVdiXnpKSy
++ rm /tmp/tmp.58C2xtHlYJ /tmp/tmp.hVdiXnpKSy
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x86gYF5MDk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.49Avxx8JGk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x86gYF5MDk
++ cat /tmp/tmp.49Avxx8JGk
++ rm /tmp/tmp.x86gYF5MDk /tmp/tmp.49Avxx8JGk
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hY5WFU2gql
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G9avvJuUv5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hY5WFU2gql
++ cat /tmp/tmp.G9avvJuUv5
++ rm /tmp/tmp.hY5WFU2gql /tmp/tmp.G9avvJuUv5
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 3Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NiHbqlDGOh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RcDE9RyrWi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NiHbqlDGOh
++ cat /tmp/tmp.RcDE9RyrWi
++ rm /tmp/tmp.NiHbqlDGOh /tmp/tmp.RcDE9RyrWi
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mJsOc6glIS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RB2iwcIBDG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mJsOc6glIS
++ cat /tmp/tmp.RB2iwcIBDG
++ rm /tmp/tmp.mJsOc6glIS /tmp/tmp.RB2iwcIBDG
++ return 0
+ [[ 1Gi == \3\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QVaG5NDRjn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.23mcz6J6f6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QVaG5NDRjn
++ cat /tmp/tmp.23mcz6J6f6
++ rm /tmp/tmp.QVaG5NDRjn /tmp/tmp.23mcz6J6f6
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 3Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 3Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=3Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 3Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 3Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QcJLqZyvo2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3BtnCY4YVA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QcJLqZyvo2
++ cat /tmp/tmp.3BtnCY4YVA
++ rm /tmp/tmp.QcJLqZyvo2 /tmp/tmp.3BtnCY4YVA
++ return 0
+ [[ 3Gi == \3\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 3Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 3Gi
+ desc 'verify autoscaling status is updated'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify autoscaling status is updated
-----------------------------------------------------------------------------------

+ check_autoscaling_status some-name 1 mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ local cluster=some-name
+ local expected_resize_count=1
+ shift 2
+ pvc_names=('mongod-data-some-name-rs0-0' 'mongod-data-some-name-rs0-1' 'mongod-data-some-name-rs0-2')
+ local pvc_names
+ local max_retry=30
+ local sleep_time=10
+ echo 'Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2'
Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ echo 'Expected resize count per PVC: 1'
Expected resize count per PVC: 1
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7936aSAR2l
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tkXF63wlgu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7936aSAR2l
++ cat /tmp/tmp.tkXF63wlgu
++ rm /tmp/tmp.7936aSAR2l /tmp/tmp.tkXF63wlgu
++ return 0
+ [[ '' -eq 1 ]]
+ [[ 0 -ge 30 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PsgLpLKT9P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zMq6J7L0CI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PsgLpLKT9P
++ cat /tmp/tmp.zMq6J7L0CI
++ rm /tmp/tmp.PsgLpLKT9P /tmp/tmp.zMq6J7L0CI
++ return 0
+ [[ 1 -eq 1 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fNXdTQVvk0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q9XcoDaWtO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fNXdTQVvk0
++ cat /tmp/tmp.q9XcoDaWtO
++ rm /tmp/tmp.fNXdTQVvk0 /tmp/tmp.q9XcoDaWtO
++ return 0
+ [[ 1 -eq 1 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sXuCDyxgWs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YamaBVghEq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sXuCDyxgWs
++ cat /tmp/tmp.YamaBVghEq
++ rm /tmp/tmp.sXuCDyxgWs /tmp/tmp.YamaBVghEq
++ return 0
+ [[ 1 -eq 1 ]]
+ desc 'wait for cluster consistency after resize'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for cluster consistency after resize
-----------------------------------------------------------------------------------

+ wait_cluster_consistency some-name 64
+ local cluster_name=some-name
+ local wait_time=64
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.b2XPlcwR2L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7WckcGpiGg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.b2XPlcwR2L
++ cat /tmp/tmp.7WckcGpiGg
++ rm /tmp/tmp.b2XPlcwR2L /tmp/tmp.7WckcGpiGg
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ desc 'verify cluster is ready'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify cluster is ready
-----------------------------------------------------------------------------------

+ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
++ mktemp
+ grep -q ready
+ local LAST_OUT=/tmp/tmp.cZaewByz52
++ mktemp
+ local LAST_ERR=/tmp/tmp.1OOjPmaSTa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cZaewByz52
+ cat /tmp/tmp.1OOjPmaSTa
+ rm /tmp/tmp.cZaewByz52 /tmp/tmp.1OOjPmaSTa
+ return 0
+ [[ -n '' ]]
+ [[ 0 == 1 ]]
+ desc 'test namespace resource quota enforcement'
+ set +o xtrace


-----------------------------------------------------------------------------------
test namespace resource quota enforcement
-----------------------------------------------------------------------------------

+ apply_resourcequota 9Gi
+ local quota=9Gi
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.llWIn4HaWo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MRrc4TIEHw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.llWIn4HaWo
++ cat /tmp/tmp.MRrc4TIEHw
++ rm /tmp/tmp.llWIn4HaWo /tmp/tmp.MRrc4TIEHw
++ return 0
+ default_sc=standard-rwo
+ echo 'Applying resourcequota for default storageclass standard-rwo with quota 9Gi'
Applying resourcequota for default storageclass standard-rwo with quota 9Gi
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/resourcequota.yml
+ sed s/STORAGECLASS/standard-rwo/
+ kubectl_bin apply -f -
+ sed s/QUOTA/9Gi/
++ mktemp
+ local LAST_OUT=/tmp/tmp.l9xNFpd8K2
++ mktemp
+ local LAST_ERR=/tmp/tmp.XhgOQTen5A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l9xNFpd8K2
resourcequota/default-storage-quota created
+ cat /tmp/tmp.XhgOQTen5A
+ rm /tmp/tmp.l9xNFpd8K2 /tmp/tmp.XhgOQTen5A
+ return 0
+ echo 'Filling disk again to trigger autoscaling with quota limit'
Filling disk again to trigger autoscaling with quota limit
+ fill_disk_to_threshold some-name-rs0-0 50 1700
+ local pod=some-name-rs0-0
+ local threshold=50
+ local size_mb=1700
+ echo 'Filling disk on some-name-rs0-0 to approximately 50% (creating 1700MB file)'
Filling disk on some-name-rs0-0 to approximately 50% (creating 1700MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=1700 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3Vjdncemnw
++ mktemp
+ local LAST_ERR=/tmp/tmp.x7w7TBxSm3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=1700 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3Vjdncemnw
+ cat /tmp/tmp.x7w7TBxSm3
+ rm /tmp/tmp.3Vjdncemnw /tmp/tmp.x7w7TBxSm3
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lStdyYm8LT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XAJcqW8Bn3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lStdyYm8LT
++ cat /tmp/tmp.XAJcqW8Bn3
++ rm /tmp/tmp.lStdyYm8LT /tmp/tmp.XAJcqW8Bn3
++ return 0
+ usage=65
+ echo 'Current disk usage on some-name-rs0-0: 65%'
Current disk usage on some-name-rs0-0: 65%
+ echo 'Waiting to verify PVC does not resize beyond quota (should stay at 3Gi)'
Waiting to verify PVC does not resize beyond quota (should stay at 3Gi)
+ sleep 30
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gApasim1Hh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DXdY5b39dL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gApasim1Hh
++ cat /tmp/tmp.DXdY5b39dL
++ rm /tmp/tmp.gApasim1Hh /tmp/tmp.DXdY5b39dL
++ return 0
+ current_size=3Gi
+ [[ 3Gi != \3\G\i ]]
+ desc 'increase quota to allow further autoscaling'
+ set +o xtrace


-----------------------------------------------------------------------------------
increase quota to allow further autoscaling
-----------------------------------------------------------------------------------

+ apply_resourcequota 15Gi
+ local quota=15Gi
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kZiroZMj9n
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s9izGmoyOy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kZiroZMj9n
++ cat /tmp/tmp.s9izGmoyOy
++ rm /tmp/tmp.kZiroZMj9n /tmp/tmp.s9izGmoyOy
++ return 0
+ default_sc=standard-rwo
+ echo 'Applying resourcequota for default storageclass standard-rwo with quota 15Gi'
Applying resourcequota for default storageclass standard-rwo with quota 15Gi
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/resourcequota.yml
+ sed s/STORAGECLASS/standard-rwo/
+ sed s/QUOTA/15Gi/
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PJfG0213Ly
++ mktemp
+ local LAST_ERR=/tmp/tmp.ilMJXj7pua
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PJfG0213Ly
resourcequota/default-storage-quota configured
+ cat /tmp/tmp.ilMJXj7pua
+ rm /tmp/tmp.PJfG0213Ly /tmp/tmp.ilMJXj7pua
+ return 0
+ wait_for_auto_resize mongod-data-some-name-rs0-0 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qHP0YlSmvG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XYXBAlZsZF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qHP0YlSmvG
++ cat /tmp/tmp.XYXBAlZsZF
++ rm /tmp/tmp.qHP0YlSmvG /tmp/tmp.XYXBAlZsZF
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PWL2qmErQh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FPTJFmwcB5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PWL2qmErQh
++ cat /tmp/tmp.FPTJFmwcB5
++ rm /tmp/tmp.PWL2qmErQh /tmp/tmp.FPTJFmwcB5
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WXapTXrktu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Defrcg9we8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WXapTXrktu
++ cat /tmp/tmp.Defrcg9we8
++ rm /tmp/tmp.WXapTXrktu /tmp/tmp.Defrcg9we8
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.STBb0WWoVH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7DW4kqoR1P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.STBb0WWoVH
++ cat /tmp/tmp.7DW4kqoR1P
++ rm /tmp/tmp.STBb0WWoVH /tmp/tmp.7DW4kqoR1P
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qjr3fetnT8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aCm4jom5cG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qjr3fetnT8
++ cat /tmp/tmp.aCm4jom5cG
++ rm /tmp/tmp.qjr3fetnT8 /tmp/tmp.aCm4jom5cG
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 4 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=5
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jNlUNLmfgl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WCeCQkHOtp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jNlUNLmfgl
++ cat /tmp/tmp.WCeCQkHOtp
++ rm /tmp/tmp.jNlUNLmfgl /tmp/tmp.WCeCQkHOtp
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 5 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=6
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ey6g7rfNDZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hMpMeprpaF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ey6g7rfNDZ
++ cat /tmp/tmp.hMpMeprpaF
++ rm /tmp/tmp.ey6g7rfNDZ /tmp/tmp.hMpMeprpaF
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 5Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AfLals2Har
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7bhkPO8Tta
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.AfLals2Har
++ cat /tmp/tmp.7bhkPO8Tta
++ rm /tmp/tmp.AfLals2Har /tmp/tmp.7bhkPO8Tta
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RU2s0Fhn81
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SJdVHygbJc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RU2s0Fhn81
++ cat /tmp/tmp.SJdVHygbJc
++ rm /tmp/tmp.RU2s0Fhn81 /tmp/tmp.SJdVHygbJc
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LS17QYxMFg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PmLhhxfueM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LS17QYxMFg
++ cat /tmp/tmp.PmLhhxfueM
++ rm /tmp/tmp.LS17QYxMFg /tmp/tmp.PmLhhxfueM
++ return 0
+ [[ 3Gi == \5\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PAI8iZR38g
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4lMjb96gDI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PAI8iZR38g
++ cat /tmp/tmp.4lMjb96gDI
++ rm /tmp/tmp.PAI8iZR38g /tmp/tmp.4lMjb96gDI
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 5Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 5Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=5Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 5Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 5Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.q7bxJTDaet
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SzsPijp6ZZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.q7bxJTDaet
++ cat /tmp/tmp.SzsPijp6ZZ
++ rm /tmp/tmp.q7bxJTDaet /tmp/tmp.SzsPijp6ZZ
++ return 0
+ [[ 5Gi == \5\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 5Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 5Gi
+ desc 'verify cluster consistency after quota test'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify cluster consistency after quota test
-----------------------------------------------------------------------------------

+ wait_cluster_consistency some-name 64
+ local cluster_name=some-name
+ local wait_time=64
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5jvPqJh73h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sBS6Gcq7UD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5jvPqJh73h
++ cat /tmp/tmp.sBS6Gcq7UD
++ rm /tmp/tmp.5jvPqJh73h /tmp/tmp.sBS6Gcq7UD
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ desc 'test maxSize limit'
+ set +o xtrace


-----------------------------------------------------------------------------------
test maxSize limit
-----------------------------------------------------------------------------------

+ apply_resourcequota 30Gi
+ local quota=30Gi
+ local default_sc
++ get_default_storageclass
++ kubectl_bin get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7hJoK8bcZc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zgiR8S5oJL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get sc -o 'jsonpath={.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7hJoK8bcZc
++ cat /tmp/tmp.zgiR8S5oJL
++ rm /tmp/tmp.7hJoK8bcZc /tmp/tmp.zgiR8S5oJL
++ return 0
+ default_sc=standard-rwo
+ echo 'Applying resourcequota for default storageclass standard-rwo with quota 30Gi'
Applying resourcequota for default storageclass standard-rwo with quota 30Gi
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/e2e-tests/pvc-auto-resize/conf/resourcequota.yml
+ sed s/STORAGECLASS/standard-rwo/
+ sed s/QUOTA/30Gi/
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hXiORRIbYO
++ mktemp
+ local LAST_ERR=/tmp/tmp.7Yp4imbAVS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hXiORRIbYO
resourcequota/default-storage-quota configured
+ cat /tmp/tmp.7Yp4imbAVS
+ rm /tmp/tmp.hXiORRIbYO /tmp/tmp.7Yp4imbAVS
+ return 0
+ echo 'Continuing to fill disk to test maxSize enforcement'
Continuing to fill disk to test maxSize enforcement
+ fill_disk_to_threshold some-name-rs0-0 50 2900
+ local pod=some-name-rs0-0
+ local threshold=50
+ local size_mb=2900
+ echo 'Filling disk on some-name-rs0-0 to approximately 50% (creating 2900MB file)'
Filling disk on some-name-rs0-0 to approximately 50% (creating 2900MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=2900 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.udB6ldgcfw
++ mktemp
+ local LAST_ERR=/tmp/tmp.7D1ZjUZgeo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=2900 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.udB6ldgcfw
+ cat /tmp/tmp.7D1ZjUZgeo
+ rm /tmp/tmp.udB6ldgcfw /tmp/tmp.7D1ZjUZgeo
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cLFIqkQy0p
+++ mktemp
++ local LAST_ERR=/tmp/tmp.isaD6X4yKl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cLFIqkQy0p
++ cat /tmp/tmp.isaD6X4yKl
++ rm /tmp/tmp.cLFIqkQy0p /tmp/tmp.isaD6X4yKl
++ return 0
+ usage=63
+ echo 'Current disk usage on some-name-rs0-0: 63%'
Current disk usage on some-name-rs0-0: 63%
+ wait_for_auto_resize mongod-data-some-name-rs0-0 7Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=7Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 7Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 7Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Srac1EAw58
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3qcAKmIdOt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Srac1EAw58
++ cat /tmp/tmp.3qcAKmIdOt
++ rm /tmp/tmp.Srac1EAw58 /tmp/tmp.3qcAKmIdOt
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iaII1GpSbm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.omzicWCo8j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iaII1GpSbm
++ cat /tmp/tmp.omzicWCo8j
++ rm /tmp/tmp.iaII1GpSbm /tmp/tmp.omzicWCo8j
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ib0vq6KW8M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.839LdWlyFf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ib0vq6KW8M
++ cat /tmp/tmp.839LdWlyFf
++ rm /tmp/tmp.ib0vq6KW8M /tmp/tmp.839LdWlyFf
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IU3Rh3G8OL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yderw2Qf40
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IU3Rh3G8OL
++ cat /tmp/tmp.yderw2Qf40
++ rm /tmp/tmp.IU3Rh3G8OL /tmp/tmp.yderw2Qf40
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fIpwX4ATTC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jw91HS7cCF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fIpwX4ATTC
++ cat /tmp/tmp.Jw91HS7cCF
++ rm /tmp/tmp.fIpwX4ATTC /tmp/tmp.Jw91HS7cCF
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 4 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=5
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ANEF4KcWdg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KTMztUGsOm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ANEF4KcWdg
++ cat /tmp/tmp.KTMztUGsOm
++ rm /tmp/tmp.ANEF4KcWdg /tmp/tmp.KTMztUGsOm
++ return 0
+ [[ 5Gi == \7\G\i ]]
+ [[ 5 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=6
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZGMslJdcoH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZK7JoBEgrL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZGMslJdcoH
++ cat /tmp/tmp.ZK7JoBEgrL
++ rm /tmp/tmp.ZGMslJdcoH /tmp/tmp.ZK7JoBEgrL
++ return 0
+ [[ 7Gi == \7\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 7Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 7Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 7Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=7Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 7Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 7Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7s49gvcevq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.maRfRLDsUz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7s49gvcevq
++ cat /tmp/tmp.maRfRLDsUz
++ rm /tmp/tmp.7s49gvcevq /tmp/tmp.maRfRLDsUz
++ return 0
+ [[ 7Gi == \7\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 7Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 7Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 7Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=7Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 7Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 7Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hnEzlfXYEM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wAhqhLj4c1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hnEzlfXYEM
++ cat /tmp/tmp.wAhqhLj4c1
++ rm /tmp/tmp.hnEzlfXYEM /tmp/tmp.wAhqhLj4c1
++ return 0
+ [[ 7Gi == \7\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 7Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 7Gi
+ fill_disk_to_threshold some-name-rs0-0 50 4000
+ local pod=some-name-rs0-0
+ local threshold=50
+ local size_mb=4000
+ echo 'Filling disk on some-name-rs0-0 to approximately 50% (creating 4000MB file)'
Filling disk on some-name-rs0-0 to approximately 50% (creating 4000MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=4000 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2il4l1V1Cl
++ mktemp
+ local LAST_ERR=/tmp/tmp.jNmKoxftHy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=4000 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2il4l1V1Cl
+ cat /tmp/tmp.jNmKoxftHy
+ rm /tmp/tmp.2il4l1V1Cl /tmp/tmp.jNmKoxftHy
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9laLmcbHFb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.99yRhrnFQP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9laLmcbHFb
++ cat /tmp/tmp.99yRhrnFQP
++ rm /tmp/tmp.9laLmcbHFb /tmp/tmp.99yRhrnFQP
++ return 0
+ usage=61
+ echo 'Current disk usage on some-name-rs0-0: 61%'
Current disk usage on some-name-rs0-0: 61%
+ wait_for_auto_resize mongod-data-some-name-rs0-0 9Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=9Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 9Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 9Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OZJc51P4Kn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QrRgCXhZhM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OZJc51P4Kn
++ cat /tmp/tmp.QrRgCXhZhM
++ rm /tmp/tmp.OZJc51P4Kn /tmp/tmp.QrRgCXhZhM
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nR8v1acZM6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.94lf0DihgZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nR8v1acZM6
++ cat /tmp/tmp.94lf0DihgZ
++ rm /tmp/tmp.nR8v1acZM6 /tmp/tmp.94lf0DihgZ
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LpP3fdGLeP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GcfmpZ84zy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LpP3fdGLeP
++ cat /tmp/tmp.GcfmpZ84zy
++ rm /tmp/tmp.LpP3fdGLeP /tmp/tmp.GcfmpZ84zy
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iaJ4FjiMkK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SZztzZOZ25
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iaJ4FjiMkK
++ cat /tmp/tmp.SZztzZOZ25
++ rm /tmp/tmp.iaJ4FjiMkK /tmp/tmp.SZztzZOZ25
++ return 0
+ [[ 9Gi == \9\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 9Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 9Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 9Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=9Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 9Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 9Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oF9VrE0FM2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VpgbH9ySZp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oF9VrE0FM2
++ cat /tmp/tmp.VpgbH9ySZp
++ rm /tmp/tmp.oF9VrE0FM2 /tmp/tmp.VpgbH9ySZp
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lDcg1N2wgM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m6wGkbWSaY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lDcg1N2wgM
++ cat /tmp/tmp.m6wGkbWSaY
++ rm /tmp/tmp.lDcg1N2wgM /tmp/tmp.m6wGkbWSaY
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NGp4mt70ad
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vFajkoaESJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NGp4mt70ad
++ cat /tmp/tmp.vFajkoaESJ
++ rm /tmp/tmp.NGp4mt70ad /tmp/tmp.vFajkoaESJ
++ return 0
+ [[ 7Gi == \9\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.et9knrc7Zm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.urHmipZeiL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.et9knrc7Zm
++ cat /tmp/tmp.urHmipZeiL
++ rm /tmp/tmp.et9knrc7Zm /tmp/tmp.urHmipZeiL
++ return 0
+ [[ 9Gi == \9\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 9Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 9Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 9Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=9Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 9Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 9Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2eLZvTuznV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.B8qAJNQlmU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2eLZvTuznV
++ cat /tmp/tmp.B8qAJNQlmU
++ rm /tmp/tmp.2eLZvTuznV /tmp/tmp.B8qAJNQlmU
++ return 0
+ [[ 9Gi == \9\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 9Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 9Gi
+ fill_disk_to_threshold some-name-rs0-0 50 5200
+ local pod=some-name-rs0-0
+ local threshold=50
+ local size_mb=5200
+ echo 'Filling disk on some-name-rs0-0 to approximately 50% (creating 5200MB file)'
Filling disk on some-name-rs0-0 to approximately 50% (creating 5200MB file)
+ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=5200 2>/dev/null || true'
++ mktemp
+ local LAST_OUT=/tmp/tmp.91DXMzPy40
++ mktemp
+ local LAST_ERR=/tmp/tmp.TUqPZDxNLT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'dd if=/dev/zero of=/data/db/fillfile bs=1M count=5200 2>/dev/null || true'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.91DXMzPy40
+ cat /tmp/tmp.TUqPZDxNLT
+ rm /tmp/tmp.91DXMzPy40 /tmp/tmp.TUqPZDxNLT
+ return 0
+ local usage
++ kubectl_bin exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HJZppfdyu7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GgWCIBfm01
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec some-name-rs0-0 -c mongod -- bash -c 'df /data/db | tail -1 | awk '\''{print $5}'\'' | sed '\''s/%//'\'''
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HJZppfdyu7
++ cat /tmp/tmp.GgWCIBfm01
++ rm /tmp/tmp.HJZppfdyu7 /tmp/tmp.GgWCIBfm01
++ return 0
+ usage=61
+ echo 'Current disk usage on some-name-rs0-0: 61%'
Current disk usage on some-name-rs0-0: 61%
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hP64W7i8sx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wi74AiyfaR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hP64W7i8sx
++ cat /tmp/tmp.wi74AiyfaR
++ rm /tmp/tmp.hP64W7i8sx /tmp/tmp.wi74AiyfaR
++ return 0
+ final_size=9Gi
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 9Gi == \9\G\i ]]
+ echo 'PVC is at 9Gi, will resize to 10Gi (maxSize)'
PVC is at 9Gi, will resize to 10Gi (maxSize)
+ wait_for_auto_resize mongod-data-some-name-rs0-0 10Gi 60 10
+ local pvc=mongod-data-some-name-rs0-0
+ local expected_size=10Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 10Gi'
Waiting for PVC mongod-data-some-name-rs0-0 to auto-resize to 10Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kH6u5NWGl4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UZWJ5frNWg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kH6u5NWGl4
++ cat /tmp/tmp.UZWJ5frNWg
++ rm /tmp/tmp.kH6u5NWGl4 /tmp/tmp.UZWJ5frNWg
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 0 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=1
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z3adQ0idtt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3HetVFVjI8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z3adQ0idtt
++ cat /tmp/tmp.3HetVFVjI8
++ rm /tmp/tmp.z3adQ0idtt /tmp/tmp.3HetVFVjI8
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 1 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=2
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CjrTPQJwX3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FO1uhQrqBs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CjrTPQJwX3
++ cat /tmp/tmp.FO1uhQrqBs
++ rm /tmp/tmp.CjrTPQJwX3 /tmp/tmp.FO1uhQrqBs
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 2 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=3
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vDx1UqVZNo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tFXm3a4MaK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vDx1UqVZNo
++ cat /tmp/tmp.tFXm3a4MaK
++ rm /tmp/tmp.vDx1UqVZNo /tmp/tmp.tFXm3a4MaK
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 3 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=4
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LISwpJcwHj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fltfOTe6Iy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LISwpJcwHj
++ cat /tmp/tmp.fltfOTe6Iy
++ rm /tmp/tmp.LISwpJcwHj /tmp/tmp.fltfOTe6Iy
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 4 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=5
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.APFkYLDiFB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.D8iBVGdhvW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.APFkYLDiFB
++ cat /tmp/tmp.D8iBVGdhvW
++ rm /tmp/tmp.APFkYLDiFB /tmp/tmp.D8iBVGdhvW
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 5 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=6
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LxCRur4yjO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6XPeN7lBnL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LxCRur4yjO
++ cat /tmp/tmp.6XPeN7lBnL
++ rm /tmp/tmp.LxCRur4yjO /tmp/tmp.6XPeN7lBnL
++ return 0
+ [[ 9Gi == \1\0\G\i ]]
+ [[ 6 -ge 60 ]]
+ echo -n .
.+ sleep 10
+ retry=7
++ get_pvc_size mongod-data-some-name-rs0-0
++ local pvc=mongod-data-some-name-rs0-0
++ kubectl_bin get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gSfW4F8kwk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.876cBpKx8V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-0 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gSfW4F8kwk
++ cat /tmp/tmp.876cBpKx8V
++ rm /tmp/tmp.gSfW4F8kwk /tmp/tmp.876cBpKx8V
++ return 0
+ [[ 10Gi == \1\0\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-0 successfully resized to 10Gi'
PVC mongod-data-some-name-rs0-0 successfully resized to 10Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-1 10Gi 60 10
+ local pvc=mongod-data-some-name-rs0-1
+ local expected_size=10Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 10Gi'
Waiting for PVC mongod-data-some-name-rs0-1 to auto-resize to 10Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-1
++ local pvc=mongod-data-some-name-rs0-1
++ kubectl_bin get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ztTTop42VB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1lx8A0ves8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-1 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ztTTop42VB
++ cat /tmp/tmp.1lx8A0ves8
++ rm /tmp/tmp.ztTTop42VB /tmp/tmp.1lx8A0ves8
++ return 0
+ [[ 10Gi == \1\0\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-1 successfully resized to 10Gi'
PVC mongod-data-some-name-rs0-1 successfully resized to 10Gi
+ wait_for_auto_resize mongod-data-some-name-rs0-2 10Gi 60 10
+ local pvc=mongod-data-some-name-rs0-2
+ local expected_size=10Gi
+ local max_retry=60
+ local sleep_time=10
+ echo 'Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 10Gi'
Waiting for PVC mongod-data-some-name-rs0-2 to auto-resize to 10Gi
+ local retry=0
++ get_pvc_size mongod-data-some-name-rs0-2
++ local pvc=mongod-data-some-name-rs0-2
++ kubectl_bin get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8pmsrNr9zx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VhIsa1PC5U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pvc mongod-data-some-name-rs0-2 -o 'jsonpath={.status.capacity.storage}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8pmsrNr9zx
++ cat /tmp/tmp.VhIsa1PC5U
++ rm /tmp/tmp.8pmsrNr9zx /tmp/tmp.VhIsa1PC5U
++ return 0
+ [[ 10Gi == \1\0\G\i ]]
+ echo

+ echo 'PVC mongod-data-some-name-rs0-2 successfully resized to 10Gi'
PVC mongod-data-some-name-rs0-2 successfully resized to 10Gi
+ desc 'verify final autoscaling status'
+ set +o xtrace


-----------------------------------------------------------------------------------
verify final autoscaling status
-----------------------------------------------------------------------------------

+ check_autoscaling_status some-name 4 mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ local cluster=some-name
+ local expected_resize_count=4
+ shift 2
+ pvc_names=('mongod-data-some-name-rs0-0' 'mongod-data-some-name-rs0-1' 'mongod-data-some-name-rs0-2')
+ local pvc_names
+ local max_retry=30
+ local sleep_time=10
+ echo 'Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2'
Checking autoscaling status for PVCs: mongod-data-some-name-rs0-0 mongod-data-some-name-rs0-1 mongod-data-some-name-rs0-2
+ echo 'Expected resize count per PVC: 4'
Expected resize count per PVC: 4
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jGrvgyVgIV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ccpuvChidL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-0.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jGrvgyVgIV
++ cat /tmp/tmp.ccpuvChidL
++ rm /tmp/tmp.jGrvgyVgIV /tmp/tmp.ccpuvChidL
++ return 0
+ [[ 4 -eq 4 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mWmSNCmJsS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o97awCoyZg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-1.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mWmSNCmJsS
++ cat /tmp/tmp.o97awCoyZg
++ rm /tmp/tmp.mWmSNCmJsS /tmp/tmp.o97awCoyZg
++ return 0
+ [[ 4 -eq 4 ]]
+ for pvc_name in "${pvc_names[@]}"
+ local retry=0
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DuBgtI8I7Z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ot6C1gm8yw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.storageAutoscaling.mongod-data-some-name-rs0-2.resizeCount}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DuBgtI8I7Z
++ cat /tmp/tmp.Ot6C1gm8yw
++ rm /tmp/tmp.DuBgtI8I7Z /tmp/tmp.Ot6C1gm8yw
++ return 0
+ [[ 4 -eq 4 ]]
+ wait_cluster_consistency some-name 64
+ local cluster_name=some-name
+ local wait_time=64
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uRCzZ1Nwf4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4W3kHbWbz3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uRCzZ1Nwf4
++ cat /tmp/tmp.4W3kHbWbz3
++ rm /tmp/tmp.uRCzZ1Nwf4 /tmp/tmp.4W3kHbWbz3
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 64 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GG8KLOOWaJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IgesUjdqGC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GG8KLOOWaJ
++ cat /tmp/tmp.IgesUjdqGC
++ rm /tmp/tmp.GG8KLOOWaJ /tmp/tmp.IgesUjdqGC
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo .OK
.OK
+ destroy pvc-auto-resize-23145
+ local namespace=pvc-auto-resize-23145
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ wc -l
++ kubectl_bin get psmdb-backup --no-headers
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dR69Ub8NMI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HC2lNvpGBq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dR69Ub8NMI
++ cat /tmp/tmp.HC2lNvpGBq
No resources found in pvc-auto-resize-23145 namespace.
++ rm /tmp/tmp.dR69Ub8NMI /tmp/tmp.HC2lNvpGBq
++ return 0
+ '[' 0 '!=' 0 ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.CqxkSoIlYD
++ mktemp
+ local LAST_ERR=/tmp/tmp.Om23poJtD9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CqxkSoIlYD
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.Om23poJtD9
+ rm /tmp/tmp.CqxkSoIlYD /tmp/tmp.Om23poJtD9
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/crd.yaml
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.crECoW091q
++ mktemp
+ local LAST_ERR=/tmp/tmp.MJGg7X0MWm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.crECoW091q
+ cat /tmp/tmp.MJGg7X0MWm
+ rm /tmp/tmp.crECoW091q /tmp/tmp.MJGg7X0MWm
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.zk3V2ZRnnz
++ mktemp
+ local LAST_ERR=/tmp/tmp.hNWy5y3KSg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zk3V2ZRnnz
+ cat /tmp/tmp.hNWy5y3KSg
+ rm /tmp/tmp.zk3V2ZRnnz /tmp/tmp.hNWy5y3KSg
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n pvc-auto-resize-23145 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodb.psmdb.percona.com/some-name patched
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.CVm5qoFGLR
++ mktemp
+ local LAST_ERR=/tmp/tmp.BTKdkGoWl5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CVm5qoFGLR
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.BTKdkGoWl5
+ rm /tmp/tmp.CVm5qoFGLR /tmp/tmp.BTKdkGoWl5
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.pVJOH7QTqY
++ mktemp
+ local LAST_ERR=/tmp/tmp.giqipNTt67
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2270/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pVJOH7QTqY
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.giqipNTt67
+ rm /tmp/tmp.pVJOH7QTqY /tmp/tmp.giqipNTt67
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lf5kBZKVR3
++ mktemp
+ local LAST_ERR=/tmp/tmp.4My7s81ZjJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Lf5kBZKVR3
+ cat /tmp/tmp.4My7s81ZjJ
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Lf5kBZKVR3
+ cat /tmp/tmp.4My7s81ZjJ
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.Lf5kBZKVR3
+ cat /tmp/tmp.4My7s81ZjJ
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.Lf5kBZKVR3
+ cat /tmp/tmp.4My7s81ZjJ
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.19.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.Lf5kBZKVR3 /tmp/tmp.4My7s81ZjJ
+ return 1
+ true
+ '[' -n '' ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pvc-auto-resize-23145
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.LkB8c0ThLn
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.NFZaxxzbJq
++ mktemp
+ local LAST_OUT=/tmp/tmp.zkOsCdKMaB
++ mktemp
+ local LAST_ERR=/tmp/tmp.jIz6tLqo3j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ local LAST_ERR=/tmp/tmp.7sUtsXkhaF
+ kubectl delete --grace-period=0 --force=true namespace pvc-auto-resize-23145
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator