++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/logs/mongod-major-upgrade-sharded.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/logs/mongod-major-upgrade-sharded.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' v1.26.15-gke.1381000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ main
+ rbac=rbac
+ '[' -n psmdb-operator ']'
+ rbac=cw-rbac
+ create_infra mongod-major-upgrade-sharded-31976
+ local ns=mongod-major-upgrade-sharded-31976
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.bZOdNIUp4y
++ mktemp
+ local LAST_ERR=/tmp/tmp.nk4MLqm1fO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bZOdNIUp4y
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.nk4MLqm1fO
+ rm /tmp/tmp.bZOdNIUp4y /tmp/tmp.nk4MLqm1fO
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.u8RyLkRchB
++ mktemp
+ local LAST_ERR=/tmp/tmp.u7nShYmEXb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.u8RyLkRchB
+ cat /tmp/tmp.u7nShYmEXb
+ rm /tmp/tmp.u8RyLkRchB /tmp/tmp.u7nShYmEXb
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.lq56z4ab8y
++ mktemp
+ local LAST_ERR=/tmp/tmp.xmSppKGRvT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lq56z4ab8y
+ cat /tmp/tmp.xmSppKGRvT
+ rm /tmp/tmp.lq56z4ab8y /tmp/tmp.xmSppKGRvT
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.MQo1KhvdLc
++ mktemp
+ local LAST_ERR=/tmp/tmp.YGNDyzx1B3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MQo1KhvdLc
+ cat /tmp/tmp.YGNDyzx1B3
+ rm /tmp/tmp.MQo1KhvdLc /tmp/tmp.YGNDyzx1B3
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.w36nIhnzu7
++ mktemp
+ local LAST_ERR=/tmp/tmp.XFn1iiMsEz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.w36nIhnzu7
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.XFn1iiMsEz
+ rm /tmp/tmp.w36nIhnzu7 /tmp/tmp.XFn1iiMsEz
+ return 0
+ check_crd_for_deletion PR-1551-e268aa9a
+ local git_tag=PR-1551-e268aa9a
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1551-e268aa9a/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6FDrogHZv2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AisSqFoWcs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6FDrogHZv2
++ cat /tmp/tmp.AisSqFoWcs
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6FDrogHZv2
++ cat /tmp/tmp.AisSqFoWcs
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.6FDrogHZv2
++ cat /tmp/tmp.AisSqFoWcs
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.6FDrogHZv2
++ cat /tmp/tmp.AisSqFoWcs
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.6FDrogHZv2 /tmp/tmp.AisSqFoWcs
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.yQzvTJn0CI
+ local LAST_OUT=/tmp/tmp.FJ6dCZd2sS
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.iqOHja8FvR
+ local LAST_ERR=/tmp/tmp.g2VqG0ONhe
+ local exit_status=0
+ local exit_status=0
+ local timeout=4
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ for i in '$(seq 0 2)'
+ set +e
+ set +e
+ kubectl get ns
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FJ6dCZd2sS
+ cat /tmp/tmp.iqOHja8FvR
+ rm /tmp/tmp.FJ6dCZd2sS /tmp/tmp.iqOHja8FvR
+ return 0
namespace "mongod-major-upgrade-sharded-22964" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yQzvTJn0CI
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.g2VqG0ONhe
+ rm /tmp/tmp.yQzvTJn0CI /tmp/tmp.g2VqG0ONhe
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qDkbC75Fd8
++ mktemp
+ local LAST_ERR=/tmp/tmp.Tfl2DsPGzk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qDkbC75Fd8
+ cat /tmp/tmp.Tfl2DsPGzk
+ rm /tmp/tmp.qDkbC75Fd8 /tmp/tmp.Tfl2DsPGzk
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.0uIy74cjDi
++ mktemp
+ local LAST_ERR=/tmp/tmp.5KvmpmALdS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0uIy74cjDi
namespace/psmdb-operator created
+ cat /tmp/tmp.5KvmpmALdS
+ rm /tmp/tmp.0uIy74cjDi /tmp/tmp.5KvmpmALdS
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mQkZb7h8Yk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rkiXZflwPP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mQkZb7h8Yk
++ cat /tmp/tmp.rkiXZflwPP
++ rm /tmp/tmp.mQkZb7h8Yk /tmp/tmp.rkiXZflwPP
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.tiy3Ie8ZHs
++ mktemp
+ local LAST_ERR=/tmp/tmp.7MiHK8YwlO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tiy3Ie8ZHs
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8" modified.
+ cat /tmp/tmp.7MiHK8YwlO
+ rm /tmp/tmp.tiy3Ie8ZHs /tmp/tmp.7MiHK8YwlO
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/mongod-major-upgrade-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SdbU3hjw4e
++ mktemp
+ local LAST_ERR=/tmp/tmp.eCpnUnmKMV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SdbU3hjw4e
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.eCpnUnmKMV
+ rm /tmp/tmp.SdbU3hjw4e /tmp/tmp.eCpnUnmKMV
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jCD2YONrTt
++ mktemp
+ local LAST_ERR=/tmp/tmp.NtYzD5cR8j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jCD2YONrTt
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.NtYzD5cR8j
+ rm /tmp/tmp.jCD2YONrTt /tmp/tmp.NtYzD5cR8j
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1551-e268aa9a") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YoPWx26I10
++ mktemp
+ local LAST_ERR=/tmp/tmp.CTEPEvd9Rf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YoPWx26I10
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.CTEPEvd9Rf
+ rm /tmp/tmp.YoPWx26I10 /tmp/tmp.CTEPEvd9Rf
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w5mRQYgUFE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qxcpkXSlLz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w5mRQYgUFE
++ cat /tmp/tmp.qxcpkXSlLz
++ rm /tmp/tmp.w5mRQYgUFE /tmp/tmp.qxcpkXSlLz
++ return 0
+ wait_pod percona-server-mongodb-operator-56586457c4-mjbl8
+ local pod=percona-server-mongodb-operator-56586457c4-mjbl8
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-56586457c4-mjbl8 to be ready.OK
+ create_namespace mongod-major-upgrade-sharded-31976
+ local namespace=mongod-major-upgrade-sharded-31976
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
++ mktemp
+ desc 'cleaned up old namespaces mongod-major-upgrade-sharded-31976'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces mongod-major-upgrade-sharded-31976
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace mongod-major-upgrade-sharded-31976 --ignore-not-found
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.MWTX5B38Zr
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ xargs kubectl delete ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.bWNOA989t0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.8kg2fTArY8
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.DyAvSy7a23
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace mongod-major-upgrade-sharded-31976 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MWTX5B38Zr
+ cat /tmp/tmp.bWNOA989t0
+ rm /tmp/tmp.MWTX5B38Zr /tmp/tmp.bWNOA989t0
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8kg2fTArY8
+ cat /tmp/tmp.DyAvSy7a23
+ rm /tmp/tmp.8kg2fTArY8 /tmp/tmp.DyAvSy7a23
+ return 0
+ kubectl_bin wait --for=delete namespace mongod-major-upgrade-sharded-31976
++ mktemp
+ local LAST_OUT=/tmp/tmp.nFqJoEkOdl
++ mktemp
+ local LAST_ERR=/tmp/tmp.bp8i3jVXlV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace mongod-major-upgrade-sharded-31976
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nFqJoEkOdl
+ cat /tmp/tmp.bp8i3jVXlV
+ rm /tmp/tmp.nFqJoEkOdl /tmp/tmp.bp8i3jVXlV
+ return 0
+ desc 'create namespace mongod-major-upgrade-sharded-31976'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace mongod-major-upgrade-sharded-31976
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace mongod-major-upgrade-sharded-31976
++ mktemp
+ local LAST_OUT=/tmp/tmp.k7d8pla8G0
++ mktemp
+ local LAST_ERR=/tmp/tmp.8eisF048po
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace mongod-major-upgrade-sharded-31976
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.k7d8pla8G0
namespace/mongod-major-upgrade-sharded-31976 created
+ cat /tmp/tmp.8eisF048po
+ rm /tmp/tmp.k7d8pla8G0 /tmp/tmp.8eisF048po
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yTHGkIOQYw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q1rQYayrjF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yTHGkIOQYw
++ cat /tmp/tmp.Q1rQYayrjF
++ rm /tmp/tmp.yTHGkIOQYw /tmp/tmp.Q1rQYayrjF
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8 --namespace=mongod-major-upgrade-sharded-31976
++ mktemp
+ local LAST_OUT=/tmp/tmp.0AdWnlleU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.qgC8BrdH0O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8 --namespace=mongod-major-upgrade-sharded-31976
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0AdWnlleU3
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1551-e268aa9a-2-cluster8" modified.
+ cat /tmp/tmp.qgC8BrdH0O
+ rm /tmp/tmp.0AdWnlleU3 /tmp/tmp.qgC8BrdH0O
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zjqin3wZHt
++ mktemp
+ local LAST_ERR=/tmp/tmp.qi7qnEaDdm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zjqin3wZHt
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.qi7qnEaDdm
+ rm /tmp/tmp.zjqin3wZHt /tmp/tmp.qi7qnEaDdm
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.3XyUubv7fR
++ mktemp
+ local LAST_ERR=/tmp/tmp.FVYYkiNaz7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3XyUubv7fR
deployment.apps/psmdb-client created
secret/some-users created
+ cat /tmp/tmp.FVYYkiNaz7
+ rm /tmp/tmp.3XyUubv7fR /tmp/tmp.FVYYkiNaz7
+ return 0
+ desc 'install version service'
+ set +o xtrace


-----------------------------------------------------------------------------------
install version service
-----------------------------------------------------------------------------------

+ cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/mongod-major-upgrade-sharded/conf/operator.main.psmdb-operator.dep.json /tmp/tmp.Pln8JJGBOx/operator.1.17.0.psmdb-operator.dep.json
+ generate_vs_json /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/mongod-major-upgrade-sharded/conf/operator.main.psmdb-operator.json /tmp/tmp.Pln8JJGBOx/operator.1.17.0.psmdb-operator.json
+ local template_path=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/mongod-major-upgrade-sharded/conf/operator.main.psmdb-operator.json
+ local target_path=/tmp/tmp.Pln8JJGBOx/operator.1.17.0.psmdb-operator.json
++ jq '.versions[0].operator="1.17.0"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1551/e2e-tests/mongod-major-upgrade-sharded/conf/operator.main.psmdb-operator.json
+ local 'version_service_source={
  "versions": [
    {
      "operator": "1.17.0",
      "product": "psmdb-operator",
      "matrix": {
        "mongod": {},
        "pmm": {
          "2.27.0": {
            "image_path": "percona/pmm-client:2.27.0",
            "image_hash": "4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",
            "status": "recommended",
            "critical": false
          }
        },
        "backup": {},
        "operator": {}
      }
    }
  ]
}'
+ for image_mongod in '${IMAGE_MONGOD_CHAIN[@]}'
++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod5.0
++ local image=perconalab/percona-server-mongodb-operator:main-mongod5.0
+++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod5.0 'mongod --version'
+++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g'
+++ local image=perconalab/percona-server-mongodb-operator:main-mongod5.0
+++ local 'cli=mongod --version'
+++ local pod_name=10851
+++ kubectl_bin -n default run 10851 --image=perconalab/percona-server-mongodb-operator:main-mongod5.0 --restart=Never --command -- sleep infinity
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nbSRvYUbDO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Pj1X3KLObH
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default run 10851 --image=perconalab/percona-server-mongodb-operator:main-mongod5.0 --restart=Never --command -- sleep infinity
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.nbSRvYUbDO
+++ cat /tmp/tmp.Pj1X3KLObH
+++ rm /tmp/tmp.nbSRvYUbDO /tmp/tmp.Pj1X3KLObH
+++ return 0
+++ kubectl_bin -n default wait --for=condition=Ready pod/10851
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ayZsFp8pMH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8MJQ4qWotO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default wait --for=condition=Ready pod/10851
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ayZsFp8pMH
+++ cat /tmp/tmp.8MJQ4qWotO
+++ rm /tmp/tmp.ayZsFp8pMH /tmp/tmp.8MJQ4qWotO
+++ return 0
++++ kubectl_bin -n default exec 10851 -- mongod --version
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.BVInGhUoiv
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.8Wpa4XFeiL
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 10851 -- mongod --version
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.BVInGhUoiv
++++ cat /tmp/tmp.8Wpa4XFeiL
++++ rm /tmp/tmp.BVInGhUoiv /tmp/tmp.8Wpa4XFeiL
++++ return 0
+++ local 'output=db version v5.0.26-22
Build Info: {
    "version": "5.0.26-22",
    "gitVersion": "565331dec31cc2affadd67913bef70906134aedc",
    "openSSLVersion": "OpenSSL 1.1.1k  FIPS 25 Mar 2021",
    "modules": [],
    "allocator": "tcmalloc",
    "environment": {
        "distarch": "x86_64",
        "target_arch": "x86_64"
    }
}'
+++ kubectl_bin -n default delete pod/10851 --grace-period=0 --force
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ogVKaDHDIZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Xn7IdpKeNo
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default delete pod/10851 --grace-period=0 --force
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ogVKaDHDIZ
+++ cat /tmp/tmp.Xn7IdpKeNo
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+++ rm /tmp/tmp.ogVKaDHDIZ /tmp/tmp.Xn7IdpKeNo
+++ return 0
+++ echo db version v5.0.26-22 Build Info: '{' '"version":' '"5.0.26-22",' '"gitVersion":' '"565331dec31cc2affadd67913bef70906134aedc",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}'
++ version_info=5.0.26-22
++ [[ ! 5.0.26-22 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]
++ echo 5.0.26-22
+ current_mongod_version=5.0.26-22
++ echo '{' '"versions":' '[' '{' '"operator":' '"1.17.0",' '"product":' '"psmdb-operator",' '"matrix":' '{' '"mongod":' '{},' '"pmm":' '{' '"2.27.0":' '{' '"image_path":' '"percona/pmm-client:2.27.0",' '"image_hash":' '"4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",' '"status":' '"recommended",' '"critical":' false '}' '},' '"backup":' '{},' '"operator":' '{}' '}' '}' ']' '}'
++ jq '.versions[0].matrix.mongod += {"5.0.26-22": {"image_path":"perconalab/percona-server-mongodb-operator:main-mongod5.0","status":"recommended"}}'
+ version_service_source='{
  "versions": [
    {
      "operator": "1.17.0",
      "product": "psmdb-operator",
      "matrix": {
        "mongod": {
          "5.0.26-22": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod5.0",
            "status": "recommended"
          }
        },
        "pmm": {
          "2.27.0": {
            "image_path": "percona/pmm-client:2.27.0",
            "image_hash": "4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",
            "status": "recommended",
            "critical": false
          }
        },
        "backup": {},
        "operator": {}
      }
    }
  ]
}'
+ for image_mongod in '${IMAGE_MONGOD_CHAIN[@]}'
++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod6.0
++ local image=perconalab/percona-server-mongodb-operator:main-mongod6.0
+++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod6.0 'mongod --version'
+++ local image=perconalab/percona-server-mongodb-operator:main-mongod6.0
+++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g'
+++ local 'cli=mongod --version'
+++ local pod_name=27162
+++ kubectl_bin -n default run 27162 --image=perconalab/percona-server-mongodb-operator:main-mongod6.0 --restart=Never --command -- sleep infinity
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vzkAJr7V9X
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FmGJ2WorjF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default run 27162 --image=perconalab/percona-server-mongodb-operator:main-mongod6.0 --restart=Never --command -- sleep infinity
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.vzkAJr7V9X
+++ cat /tmp/tmp.FmGJ2WorjF
+++ rm /tmp/tmp.vzkAJr7V9X /tmp/tmp.FmGJ2WorjF
+++ return 0
+++ kubectl_bin -n default wait --for=condition=Ready pod/27162
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1jRezN7Ri6
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.B1GNxoB7nw
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default wait --for=condition=Ready pod/27162
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.1jRezN7Ri6
+++ cat /tmp/tmp.B1GNxoB7nw
+++ rm /tmp/tmp.1jRezN7Ri6 /tmp/tmp.B1GNxoB7nw
+++ return 0
++++ kubectl_bin -n default exec 27162 -- mongod --version
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.m8ZX2ckJv2
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.7fpiPf811u
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 27162 -- mongod --version
++++ exit_status=1
++++ set -e
++++ '[' 1 '!=' 0 -a -n 1 ']'
++++ cat /tmp/tmp.m8ZX2ckJv2
++++ cat /tmp/tmp.7fpiPf811u
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
++++ sleep 0
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 27162 -- mongod --version
++++ exit_status=1
++++ set -e
++++ '[' 1 '!=' 0 -a -n 1 ']'
++++ cat /tmp/tmp.m8ZX2ckJv2
++++ cat /tmp/tmp.7fpiPf811u
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
++++ sleep 4
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 27162 -- mongod --version
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.m8ZX2ckJv2
++++ cat /tmp/tmp.7fpiPf811u
++++ rm /tmp/tmp.m8ZX2ckJv2 /tmp/tmp.7fpiPf811u
++++ return 0
+++ local 'output=db version v6.0.15-12
Build Info: {
    "version": "6.0.15-12",
    "gitVersion": "2c4ff0c994742506096fae92dc182d61380c2854",
    "openSSLVersion": "OpenSSL 1.1.1k  FIPS 25 Mar 2021",
    "modules": [],
    "proFeatures": [],
    "allocator": "tcmalloc",
    "environment": {
        "distarch": "x86_64",
        "target_arch": "x86_64"
    }
}'
+++ kubectl_bin -n default delete pod/27162 --grace-period=0 --force
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tDnpJDnGff
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.33WCQ5tVJm
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default delete pod/27162 --grace-period=0 --force
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tDnpJDnGff
+++ cat /tmp/tmp.33WCQ5tVJm
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+++ rm /tmp/tmp.tDnpJDnGff /tmp/tmp.33WCQ5tVJm
+++ return 0
+++ echo db version v6.0.15-12 Build Info: '{' '"version":' '"6.0.15-12",' '"gitVersion":' '"2c4ff0c994742506096fae92dc182d61380c2854",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}'
++ version_info=6.0.15-12
++ [[ ! 6.0.15-12 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]
++ echo 6.0.15-12
+ current_mongod_version=6.0.15-12
++ echo '{' '"versions":' '[' '{' '"operator":' '"1.17.0",' '"product":' '"psmdb-operator",' '"matrix":' '{' '"mongod":' '{' '"5.0.26-22":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod5.0",' '"status":' '"recommended"' '}' '},' '"pmm":' '{' '"2.27.0":' '{' '"image_path":' '"percona/pmm-client:2.27.0",' '"image_hash":' '"4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",' '"status":' '"recommended",' '"critical":' false '}' '},' '"backup":' '{},' '"operator":' '{}' '}' '}' ']' '}'
++ jq '.versions[0].matrix.mongod += {"6.0.15-12": {"image_path":"perconalab/percona-server-mongodb-operator:main-mongod6.0","status":"recommended"}}'
+ version_service_source='{
  "versions": [
    {
      "operator": "1.17.0",
      "product": "psmdb-operator",
      "matrix": {
        "mongod": {
          "5.0.26-22": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod5.0",
            "status": "recommended"
          },
          "6.0.15-12": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod6.0",
            "status": "recommended"
          }
        },
        "pmm": {
          "2.27.0": {
            "image_path": "percona/pmm-client:2.27.0",
            "image_hash": "4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",
            "status": "recommended",
            "critical": false
          }
        },
        "backup": {},
        "operator": {}
      }
    }
  ]
}'
+ for image_mongod in '${IMAGE_MONGOD_CHAIN[@]}'
++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod7.0
++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod7.0 'mongod --version'
+++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ local 'cli=mongod --version'
+++ local pod_name=229
+++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g'
+++ kubectl_bin -n default run 229 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GXueF3Awvb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.urPRZg94q6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default run 229 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GXueF3Awvb
+++ cat /tmp/tmp.urPRZg94q6
+++ rm /tmp/tmp.GXueF3Awvb /tmp/tmp.urPRZg94q6
+++ return 0
+++ kubectl_bin -n default wait --for=condition=Ready pod/229
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xerzdOZQ86
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qG2IOaNy3Q
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default wait --for=condition=Ready pod/229
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.xerzdOZQ86
+++ cat /tmp/tmp.qG2IOaNy3Q
+++ rm /tmp/tmp.xerzdOZQ86 /tmp/tmp.qG2IOaNy3Q
+++ return 0
++++ kubectl_bin -n default exec 229 -- mongod --version
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.3nHENDMEFF
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.AiZz8n23n3
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 229 -- mongod --version
++++ exit_status=1
++++ set -e
++++ '[' 1 '!=' 0 -a -n 1 ']'
++++ cat /tmp/tmp.3nHENDMEFF
++++ cat /tmp/tmp.AiZz8n23n3
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
++++ sleep 0
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 229 -- mongod --version
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.3nHENDMEFF
++++ cat /tmp/tmp.AiZz8n23n3
++++ rm /tmp/tmp.3nHENDMEFF /tmp/tmp.AiZz8n23n3
++++ return 0
+++ local 'output=db version v7.0.11-6
Build Info: {
    "version": "7.0.11-6",
    "gitVersion": "0818dfdcc4349542e73dd63f56ab35f92498a115",
    "openSSLVersion": "OpenSSL 1.1.1k  FIPS 25 Mar 2021",
    "modules": [],
    "proFeatures": [],
    "allocator": "tcmalloc",
    "environment": {
        "distarch": "x86_64",
        "target_arch": "x86_64"
    }
}'
+++ kubectl_bin -n default delete pod/229 --grace-period=0 --force
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UOFbvaIzjm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AceLpWvLW6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default delete pod/229 --grace-period=0 --force
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.UOFbvaIzjm
+++ cat /tmp/tmp.AceLpWvLW6
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+++ rm /tmp/tmp.UOFbvaIzjm /tmp/tmp.AceLpWvLW6
+++ return 0
+++ echo db version v7.0.11-6 Build Info: '{' '"version":' '"7.0.11-6",' '"gitVersion":' '"0818dfdcc4349542e73dd63f56ab35f92498a115",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}'
++ version_info=7.0.11-6
++ [[ ! 7.0.11-6 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]
++ echo 7.0.11-6
+ current_mongod_version=7.0.11-6
++ jq '.versions[0].matrix.mongod += {"7.0.11-6": {"image_path":"perconalab/percona-server-mongodb-operator:main-mongod7.0","status":"recommended"}}'
++ echo '{' '"versions":' '[' '{' '"operator":' '"1.17.0",' '"product":' '"psmdb-operator",' '"matrix":' '{' '"mongod":' '{' '"5.0.26-22":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod5.0",' '"status":' '"recommended"' '},' '"6.0.15-12":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod6.0",' '"status":' '"recommended"' '}' '},' '"pmm":' '{' '"2.27.0":' '{' '"image_path":' '"percona/pmm-client:2.27.0",' '"image_hash":' '"4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",' '"status":' '"recommended",' '"critical":' false '}' '},' '"backup":' '{},' '"operator":' '{}' '}' '}' ']' '}'
+ version_service_source='{
  "versions": [
    {
      "operator": "1.17.0",
      "product": "psmdb-operator",
      "matrix": {
        "mongod": {
          "5.0.26-22": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod5.0",
            "status": "recommended"
          },
          "6.0.15-12": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod6.0",
            "status": "recommended"
          },
          "7.0.11-6": {
            "image_path": "perconalab/percona-server-mongodb-operator:main-mongod7.0",
            "status": "recommended"
          }
        },
        "pmm": {
          "2.27.0": {
            "image_path": "percona/pmm-client:2.27.0",
            "image_hash": "4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",
            "status": "recommended",
            "critical": false
          }
        },
        "backup": {},
        "operator": {}
      }
    }
  ]
}'
++ echo '{' '"versions":' '[' '{' '"operator":' '"1.17.0",' '"product":' '"psmdb-operator",' '"matrix":' '{' '"mongod":' '{' '"5.0.26-22":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod5.0",' '"status":' '"recommended"' '},' '"6.0.15-12":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod6.0",' '"status":' '"recommended"' '},' '"7.0.11-6":' '{' '"image_path":' '"perconalab/percona-server-mongodb-operator:main-mongod7.0",' '"status":' '"recommended"' '}' '},' '"pmm":' '{' '"2.27.0":' '{' '"image_path":' '"percona/pmm-client:2.27.0",' '"image_hash":' '"4f4d4508afe4ef3b5d72e2ebec6485be84204902b0b2b23c3a9e7c2fe4726884",' '"status":' '"recommended",' '"critical":' false '}' '},' '"backup":' '{},' '"operator":' '{}' '}' '}' ']' '}'
+++ get_pbm_version perconalab/percona-server-mongodb-operator:main-backup
+++ local image=perconalab/percona-server-mongodb-operator:main-backup
++++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-backup 'pbm-agent version'
++++ local image=perconalab/percona-server-mongodb-operator:main-backup
++++ local 'cli=pbm-agent version'
++++ /usr/bin/sed -r 's/^Version:\ (([0-9]+\.){2}[0-9]+)\ .*/\1/g'
++++ local pod_name=1512
++++ kubectl_bin -n default run 1512 --image=perconalab/percona-server-mongodb-operator:main-backup --restart=Never --command -- sleep infinity
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.XUl9MLVKA0
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.FYo6uhJyD5
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default run 1512 --image=perconalab/percona-server-mongodb-operator:main-backup --restart=Never --command -- sleep infinity
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.XUl9MLVKA0
++++ cat /tmp/tmp.FYo6uhJyD5
++++ rm /tmp/tmp.XUl9MLVKA0 /tmp/tmp.FYo6uhJyD5
++++ return 0
++++ kubectl_bin -n default wait --for=condition=Ready pod/1512
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.KvESYxWAJK
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.GAKEgcJlrt
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default wait --for=condition=Ready pod/1512
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.KvESYxWAJK
++++ cat /tmp/tmp.GAKEgcJlrt
++++ rm /tmp/tmp.KvESYxWAJK /tmp/tmp.GAKEgcJlrt
++++ return 0
+++++ kubectl_bin -n default exec 1512 -- pbm-agent version
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.xfAZWu0NxX
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.P7nR7U0QFB
+++++ local exit_status=0
+++++ local timeout=4
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl -n default exec 1512 -- pbm-agent version
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.xfAZWu0NxX
+++++ cat /tmp/tmp.P7nR7U0QFB
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
+++++ sleep 0
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl -n default exec 1512 -- pbm-agent version
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.xfAZWu0NxX
+++++ cat /tmp/tmp.P7nR7U0QFB
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
+++++ sleep 4
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl -n default exec 1512 -- pbm-agent version
+++++ exit_status=1
+++++ set -e
+++++ '[' 1 '!=' 0 -a -n 1 ']'
+++++ cat /tmp/tmp.xfAZWu0NxX
+++++ cat /tmp/tmp.P7nR7U0QFB
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
+++++ sleep 8
+++++ cat /tmp/tmp.xfAZWu0NxX
+++++ cat /tmp/tmp.P7nR7U0QFB
Error from server: error dialing backend: dial tcp 10.215.0.5:10250: i/o timeout
+++++ rm /tmp/tmp.xfAZWu0NxX /tmp/tmp.P7nR7U0QFB
+++++ return 1
++++ local output=
++++ kubectl_bin -n default delete pod/1512 --grace-period=0 --force
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.3qnXfrqtUh
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.YMQYk3m3TO
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default delete pod/1512 --grace-period=0 --force
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.3qnXfrqtUh
++++ cat /tmp/tmp.YMQYk3m3TO
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
++++ rm /tmp/tmp.3qnXfrqtUh /tmp/tmp.YMQYk3m3TO
++++ return 0
++++ echo
+++ local version_info=
+++ [[ ! '' =~ ^([0-9]+\.){2}[0-9]+$ ]]
+++ printf 'No pbm version obtained from %s. Exiting' perconalab/percona-server-mongodb-operator:main-backup
+++ exit 1
++ jq '.versions[0].matrix.backup += {"No' pbm version obtained from perconalab/percona-server-mongodb-operator:main-backup. 'Exiting": {"image_path":"perconalab/percona-server-mongodb-operator:main-backup","status":"recommended"}}'
jq: error: syntax error, unexpected $end, expecting QQSTRING_TEXT or QQSTRING_INTERP_START or QQSTRING_END (Unix shell quoting issues?) at <top-level>, line 1:
.versions[0].matrix.backup += {"No                                
jq: 1 compile error
+ version_service_source=