Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/logs/liveness.log
grep: warning: stray \ before -
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1
+ create_infra liveness-17640
+ local ns=liveness-17640
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.55mIRbHr5b
++ mktemp
+ local LAST_ERR=/tmp/tmp.TnMyL3RTjI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.55mIRbHr5b
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.TnMyL3RTjI
+ rm /tmp/tmp.55mIRbHr5b /tmp/tmp.TnMyL3RTjI
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.qAks5fwk6Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.hZbo5ivj71
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qAks5fwk6Q
+ cat /tmp/tmp.hZbo5ivj71
+ rm /tmp/tmp.qAks5fwk6Q /tmp/tmp.hZbo5ivj71
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.O8lL7duAWw
++ mktemp
+ local LAST_ERR=/tmp/tmp.jKdlbTn0rj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.O8lL7duAWw
+ cat /tmp/tmp.jKdlbTn0rj
+ rm /tmp/tmp.O8lL7duAWw /tmp/tmp.jKdlbTn0rj
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.N4SrDsR7Uk
++ mktemp
+ local LAST_ERR=/tmp/tmp.LqoaPaUNlM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N4SrDsR7Uk
+ cat /tmp/tmp.LqoaPaUNlM
+ rm /tmp/tmp.N4SrDsR7Uk /tmp/tmp.LqoaPaUNlM
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Py4SXR5MR
++ mktemp
+ local LAST_ERR=/tmp/tmp.umxjorUxsN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2Py4SXR5MR
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.umxjorUxsN
+ rm /tmp/tmp.2Py4SXR5MR /tmp/tmp.umxjorUxsN
+ return 0
+ check_crd_for_deletion PR-2152-4461031b
+ local git_tag=PR-2152-4461031b
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2152-4461031b/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5QTBGeXDOb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dtt9QcZN9X
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5QTBGeXDOb
++ cat /tmp/tmp.Dtt9QcZN9X
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5QTBGeXDOb
++ cat /tmp/tmp.Dtt9QcZN9X
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.5QTBGeXDOb
++ cat /tmp/tmp.Dtt9QcZN9X
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.5QTBGeXDOb
++ cat /tmp/tmp.Dtt9QcZN9X
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.5QTBGeXDOb /tmp/tmp.Dtt9QcZN9X
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.0qdF6dYBDM
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
+ local LAST_OUT=/tmp/tmp.FDR2rl7tBZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.QIqHA15Ltl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.5zxBFx2imF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0qdF6dYBDM
+ cat /tmp/tmp.QIqHA15Ltl
+ rm /tmp/tmp.0qdF6dYBDM /tmp/tmp.QIqHA15Ltl
+ return 0
namespace "liveness-16646" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FDR2rl7tBZ
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.5zxBFx2imF
+ rm /tmp/tmp.FDR2rl7tBZ /tmp/tmp.5zxBFx2imF
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.44H6Lir5xq
++ mktemp
+ local LAST_ERR=/tmp/tmp.RFbx1GPTtA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.44H6Lir5xq
+ cat /tmp/tmp.RFbx1GPTtA
+ rm /tmp/tmp.44H6Lir5xq /tmp/tmp.RFbx1GPTtA
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZS8YzBcku0
++ mktemp
+ local LAST_ERR=/tmp/tmp.28QxQ0e1Rq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZS8YzBcku0
namespace/psmdb-operator created
+ cat /tmp/tmp.28QxQ0e1Rq
+ rm /tmp/tmp.ZS8YzBcku0 /tmp/tmp.28QxQ0e1Rq
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nLPsOJq9zP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uurDdqDZ99
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nLPsOJq9zP
++ cat /tmp/tmp.uurDdqDZ99
++ rm /tmp/tmp.nLPsOJq9zP /tmp/tmp.uurDdqDZ99
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qbQUiniM2U
++ mktemp
+ local LAST_ERR=/tmp/tmp.sf9ayuHxs4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qbQUiniM2U
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4" modified.
+ cat /tmp/tmp.sf9ayuHxs4
+ rm /tmp/tmp.qbQUiniM2U /tmp/tmp.sf9ayuHxs4
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2152-4461031b'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-2152-4461031b
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.pca1sZ676V
++ mktemp
+ local LAST_ERR=/tmp/tmp.OkOVIECBAj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pca1sZ676V
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.OkOVIECBAj
+ rm /tmp/tmp.pca1sZ676V /tmp/tmp.OkOVIECBAj
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.crTYC3CkEf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ltt3HXVa8t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.crTYC3CkEf
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.Ltt3HXVa8t
+ rm /tmp/tmp.crTYC3CkEf /tmp/tmp.Ltt3HXVa8t
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-2152-4461031b") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lnIy0gNthr
++ mktemp
+ local LAST_ERR=/tmp/tmp.IfpEvEhRDx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lnIy0gNthr
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.IfpEvEhRDx
+ rm /tmp/tmp.lnIy0gNthr /tmp/tmp.IfpEvEhRDx
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ulC6mC0ShI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l2PJz9gMRN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ulC6mC0ShI
++ cat /tmp/tmp.l2PJz9gMRN
++ rm /tmp/tmp.ulC6mC0ShI /tmp/tmp.l2PJz9gMRN
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-78c57f9588-br5r5
+ local pod=percona-server-mongodb-operator-78c57f9588-br5r5
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-78c57f9588-br5r5 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+ grep 'Manager starting up'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3cpwsQqj7q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZiZVAtCnaY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3cpwsQqj7q
++ cat /tmp/tmp.ZiZVAtCnaY
++ rm /tmp/tmp.3cpwsQqj7q /tmp/tmp.ZiZVAtCnaY
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-78c57f9588-br5r5
++ mktemp
+ local LAST_OUT=/tmp/tmp.ostzbZg0Kx
++ mktemp
+ local LAST_ERR=/tmp/tmp.nL49C79J2C
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-78c57f9588-br5r5
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ostzbZg0Kx
+ cat /tmp/tmp.nL49C79J2C
+ rm /tmp/tmp.ostzbZg0Kx /tmp/tmp.nL49C79J2C
+ return 0
2025-12-18T18:26:04.787Z	INFO	setup	Manager starting up	{"gitCommit": "4461031b25c11e5b29d77d465c058b936127aac2", "gitBranch": "PR-2152-4461031b", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"}
+ create_namespace liveness-17640
+ local namespace=liveness-17640
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces liveness-17640'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces liveness-17640
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace liveness-17640 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.dUt0fr2ndu
+ local LAST_OUT=/tmp/tmp.gqTbNYLm88
egrep: warning: egrep is obsolescent; using grep -E
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.BbOFor7Jwh
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.LnQIHHF5Ne
++ seq 0 2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace liveness-17640 --ignore-not-found
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dUt0fr2ndu
+ cat /tmp/tmp.LnQIHHF5Ne
+ rm /tmp/tmp.dUt0fr2ndu /tmp/tmp.LnQIHHF5Ne
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gqTbNYLm88
+ cat /tmp/tmp.BbOFor7Jwh
+ rm /tmp/tmp.gqTbNYLm88 /tmp/tmp.BbOFor7Jwh
+ return 0
+ kubectl_bin wait --for=delete namespace liveness-17640
++ mktemp
+ local LAST_OUT=/tmp/tmp.hkPTQPX4Sf
++ mktemp
+ local LAST_ERR=/tmp/tmp.RIiPoq8qZK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace liveness-17640
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hkPTQPX4Sf
+ cat /tmp/tmp.RIiPoq8qZK
+ rm /tmp/tmp.hkPTQPX4Sf /tmp/tmp.RIiPoq8qZK
+ return 0
+ desc 'create namespace liveness-17640'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace liveness-17640
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace liveness-17640
++ mktemp
+ local LAST_OUT=/tmp/tmp.SiAoD8aBU5
++ mktemp
+ local LAST_ERR=/tmp/tmp.hI94Pi1xRj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace liveness-17640
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SiAoD8aBU5
namespace/liveness-17640 created
+ cat /tmp/tmp.hI94Pi1xRj
+ rm /tmp/tmp.SiAoD8aBU5 /tmp/tmp.hI94Pi1xRj
+ return 0
+ set_kube_ctx liveness-17640
+ local namespace=liveness-17640
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.alE9iAvliZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EfDkG6SICr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.alE9iAvliZ
++ cat /tmp/tmp.EfDkG6SICr
++ rm /tmp/tmp.alE9iAvliZ /tmp/tmp.EfDkG6SICr
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4 --namespace=liveness-17640
++ mktemp
+ local LAST_OUT=/tmp/tmp.1FdvBPabG2
++ mktemp
+ local LAST_ERR=/tmp/tmp.mVXJqbYfXF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4 --namespace=liveness-17640
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1FdvBPabG2
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2152-4461031b-2-cluster4" modified.
+ cat /tmp/tmp.mVXJqbYfXF
+ rm /tmp/tmp.1FdvBPabG2 /tmp/tmp.mVXJqbYfXF
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fmpxix6qXH
++ mktemp
+ local LAST_ERR=/tmp/tmp.Qd0cYvyK2J
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fmpxix6qXH
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.Qd0cYvyK2J
+ rm /tmp/tmp.fmpxix6qXH /tmp/tmp.Qd0cYvyK2J
+ return 0
+ cluster=liveness
+ desc 'create first PSMDB cluster liveness'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster liveness
-----------------------------------------------------------------------------------

+ spinup_psmdb liveness-rs0 /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0.yml
+ local cluster=liveness-rs0
+ local config=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0.yml
+ local size=3
+ desc 'create first PSMDB cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2152-4461031b"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.KuF97ghx9F
++ mktemp
+ local LAST_ERR=/tmp/tmp.EP4vRbYDhu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KuF97ghx9F
perconaservermongodb.psmdb.percona.com/liveness created
+ cat /tmp/tmp.EP4vRbYDhu
+ rm /tmp/tmp.KuF97ghx9F /tmp/tmp.EP4vRbYDhu
+ return 0
+ desc 'check if Pod is started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod is started
-----------------------------------------------------------------------------------

+ wait_for_running liveness-rs0 3
+ local name=liveness-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=liveness
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod liveness-rs0-0
+ local pod=liveness-rs0-0
+ set +o xtrace
waiting for pod/liveness-rs0-0 to be ready...................OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod liveness-rs0-1
+ local pod=liveness-rs0-1
+ set +o xtrace
waiting for pod/liveness-rs0-1 to be ready..................OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PFywGNhCCz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.khOuqVc5SW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PFywGNhCCz
++ cat /tmp/tmp.khOuqVc5SW
++ rm /tmp/tmp.PFywGNhCCz /tmp/tmp.khOuqVc5SW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod liveness-rs0-2
+ local pod=liveness-rs0-2
+ set +o xtrace
waiting for pod/liveness-rs0-2 to be ready..................OK
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jqouCPHGTq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9jkSg1nRsu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jqouCPHGTq
++ cat /tmp/tmp.9jkSg1nRsu
++ rm /tmp/tmp.jqouCPHGTq /tmp/tmp.9jkSg1nRsu
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mDc7PPDxvY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.W5100dmps7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mDc7PPDxvY
++ cat /tmp/tmp.W5100dmps7
++ rm /tmp/tmp.mDc7PPDxvY /tmp/tmp.W5100dmps7
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 20
+ compare_kubectl statefulset/liveness-rs0
+ local resource=statefulset/liveness-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml
+ local new_result=/tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/liveness-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("liveness-17640", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.gguyV0UiAu
++ mktemp
+ local LAST_ERR=/tmp/tmp.Iq7D0Cc7c4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/liveness-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gguyV0UiAu
+ cat /tmp/tmp.Iq7D0Cc7c4
+ rm /tmp/tmp.gguyV0UiAu /tmp/tmp.Iq7D0Cc7c4
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ log 'compare_kubectl: statefulset/liveness-rs0 OK'
+ set +o xtrace
[2025-12-18T18:29:27+0000] compare_kubectl: statefulset/liveness-rs0 OK
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' userAdmin:userAdmin123456@liveness-rs0.liveness-17640
+ local 'command=db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})'
+ local uri=userAdmin:userAdmin123456@liveness-rs0.liveness-17640
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ userAdmin:userAdmin123456@liveness-rs0.liveness-17640 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7DkpZjinOM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HkoH4mgFQQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7DkpZjinOM
++ cat /tmp/tmp.HkoH4mgFQQ
++ rm /tmp/tmp.7DkpZjinOM /tmp/tmp.HkoH4mgFQQ
++ return 0
+ local client_container=psmdb-client-696897d69b-5zwg5
+ kubectl_bin exec psmdb-client-696897d69b-5zwg5 -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@liveness-rs0.liveness-17640.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FfYem8qwmk
++ mktemp
+ local LAST_ERR=/tmp/tmp.wD6AIDDB3Y
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5zwg5 -- bash -c 'printf '\''db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@liveness-rs0.liveness-17640.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FfYem8qwmk
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://liveness-rs0-0.liveness-rs0.liveness-17640.svc.cluster.local:27017,liveness-rs0-1.liveness-rs0.liveness-17640.svc.cluster.local:27017,liveness-rs0-2.liveness-rs0.liveness-17640.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("a2e6b3a2-1604-4c12-b756-e51ed08d9ffd") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.wD6AIDDB3Y
+ rm /tmp/tmp.FfYem8qwmk /tmp/tmp.wD6AIDDB3Y
+ return 0
+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@liveness-rs0.liveness-17640
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@liveness-rs0.liveness-17640
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local replica_set=rs0
+ [[ myApp:myPass@liveness-rs0.liveness-17640 == *cfg* ]]
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u21xIX6dVH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Pe9HAJS3OO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.u21xIX6dVH
++ cat /tmp/tmp.Pe9HAJS3OO
++ rm /tmp/tmp.u21xIX6dVH /tmp/tmp.Pe9HAJS3OO
++ return 0
+ local client_container=psmdb-client-696897d69b-5zwg5
+ kubectl_bin exec psmdb-client-696897d69b-5zwg5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@liveness-rs0.liveness-17640.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.aqkRfq8YU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.EmqAkURBET
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl exec psmdb-client-696897d69b-5zwg5 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@liveness-rs0.liveness-17640.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aqkRfq8YU3
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://liveness-rs0-2.liveness-rs0.liveness-17640.svc.cluster.local:27017,liveness-rs0-0.liveness-rs0.liveness-17640.svc.cluster.local:27017,liveness-rs0-1.liveness-rs0.liveness-17640.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5a930c21-7c1b-45a6-accd-a90e09ff6402") }
Percona Server for MongoDB server version: v8.0.16-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.EmqAkURBET
+ rm /tmp/tmp.aqkRfq8YU3 /tmp/tmp.EmqAkURBET
+ return 0
+ desc 'check if statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/liveness-rs0
+ local resource=statefulset/liveness-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml
+ local new_result=/tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/liveness-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("liveness-17640", "NAME_SPACE") |
++ mktemp
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.97j3vFFU78
++ mktemp
+ local LAST_ERR=/tmp/tmp.0wwt0sRLp6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get -o yaml statefulset/liveness-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.97j3vFFU78
+ cat /tmp/tmp.0wwt0sRLp6
+ rm /tmp/tmp.97j3vFFU78 /tmp/tmp.0wwt0sRLp6
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ version_gt 1.22
++ echo '1.31 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/compare/statefulset_liveness-rs0.yml /tmp/tmp.ACydqEal6a/statefulset_liveness-rs0.yml
+ log 'compare_kubectl: statefulset/liveness-rs0 OK'
+ set +o xtrace
[2025-12-18T18:29:34+0000] compare_kubectl: statefulset/liveness-rs0 OK
+ set +e
+ kubectl exec -it liveness-rs0-0 -- bash -c '/opt/percona/mongodb-healthcheck k8s liveness --ssl'
Defaulted container "mongod" out of: mongod, mongo-init (init)
Unable to use a TTY - input is not a terminal or the right kind of file
2025-12-18T18:29:35.785Z	INFO	Running mongodb-healthcheck	{"commit": "4461031b25c11e5b29d77d465c058b936127aac2", "branch": "PR-2152-4461031b"}
2025-12-18T18:29:35.786Z	INFO	Running Kubernetes liveness check for	{"component": "mongod"}
2025-12-18T18:29:35.786Z	DEBUG	Connecting to mongodb	{"hosts": ["localhost:27017"], "ssl": true, "ssl_insecure": false}
2025-12-18T18:29:35.786Z	DEBUG	Enabling authentication for session	{"user": "clusterMonitor"}
2025-12-18T18:29:55.788Z	ERROR	Failed to perform check	{"error": "member failed Kubernetes liveness check: connection error: filed to dial mongo: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\") }, ] }", "errorVerbose": "server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: localhost:27017, Type: Unknown, Last error: tls: failed to verify certificate: x509: certificate signed by unknown authority (possibly because of \"crypto/rsa: verification error\" while trying to verify candidate authority certificate \"Root CA\") }, ] }\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db/db.go:48\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck.HealthCheckMongodLiveness\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck/health.go:62\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool.(*App).Run\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool/tool.go:95\nmain.main\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/main.go:45\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:285\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfiled to dial mongo\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db/db.go:50\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck.HealthCheckMongodLiveness\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck/health.go:62\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool.(*App).Run\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool/tool.go:95\nmain.main\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/main.go:45\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:285\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nconnection error\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck.HealthCheckMongodLiveness\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck/health.go:64\ngithub.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool.(*App).Run\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool/tool.go:95\nmain.main\n\t/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/main.go:45\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:285\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nmember failed Kubernetes liveness check"}
main.main
	/go/src/github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/main.go:46
runtime.main
	/usr/local/go/src/runtime/proc.go:285
command terminated with exit code 1
+ set -e
+ sleep 20
+ wait_for_running liveness-rs0 3
+ local name=liveness-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=liveness
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod liveness-rs0-0
+ local pod=liveness-rs0-0
+ set +o xtrace
waiting for pod/liveness-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod liveness-rs0-1
+ local pod=liveness-rs0-1
+ set +o xtrace
waiting for pod/liveness-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sJ92KzSr85
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oORc4kuZGB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sJ92KzSr85
++ cat /tmp/tmp.oORc4kuZGB
++ rm /tmp/tmp.sJ92KzSr85 /tmp/tmp.oORc4kuZGB
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod liveness-rs0-2
+ local pod=liveness-rs0-2
+ set +o xtrace
waiting for pod/liveness-rs0-2 to be ready.OK
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gDcDrZ7iIr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5OUHLmyLif
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gDcDrZ7iIr
++ cat /tmp/tmp.5OUHLmyLif
++ rm /tmp/tmp.gDcDrZ7iIr /tmp/tmp.5OUHLmyLif
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dlp7uaMNS8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M6H1T4a9ao
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Dlp7uaMNS8
++ cat /tmp/tmp.M6H1T4a9ao
++ rm /tmp/tmp.Dlp7uaMNS8 /tmp/tmp.M6H1T4a9ao
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ desc 'check liveness logs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check liveness logs
-----------------------------------------------------------------------------------

++ kubectl_bin exec -it liveness-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs'
++ grep -c mongodb-healthcheck.log
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n5uyMF16OY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.93zyZ9bRS6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec -it liveness-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n5uyMF16OY
++ cat /tmp/tmp.93zyZ9bRS6
Defaulted container "mongod" out of: mongod, mongo-init (init)
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.n5uyMF16OY /tmp/tmp.93zyZ9bRS6
++ return 0
+ current_log_file=1
+ [[ 1 != \1 ]]
++ kubectl_bin exec -it liveness-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs'
++ grep -c 'mongodb-healthcheck-.*\.log\.gz'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QB6B1fGMDc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eDWyaOxWlA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl exec -it liveness-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QB6B1fGMDc
++ cat /tmp/tmp.eDWyaOxWlA
Defaulted container "mongod" out of: mongod, mongo-init (init)
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.QB6B1fGMDc /tmp/tmp.eDWyaOxWlA
++ return 0
+ rotated_log_files=2
+ [[ 2 -lt 1 ]]
+ desc 'change liveness config'
+ set +o xtrace


-----------------------------------------------------------------------------------
change liveness config
-----------------------------------------------------------------------------------

+ postfix=-changed
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0-changed.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0-changed.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2152/e2e-tests/liveness/conf/liveness-rs0-changed.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-2152-4461031b"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.zwWxTahpUr
++ mktemp
+ local LAST_ERR=/tmp/tmp.RrlImx9Jm7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zwWxTahpUr
perconaservermongodb.psmdb.percona.com/liveness configured
+ cat /tmp/tmp.RrlImx9Jm7
+ rm /tmp/tmp.zwWxTahpUr /tmp/tmp.RrlImx9Jm7
+ return 0
+ sleep 20
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running liveness-rs0 3
+ local name=liveness-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=liveness
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod liveness-rs0-0
+ local pod=liveness-rs0-0
+ set +o xtrace
waiting for pod/liveness-rs0-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod liveness-rs0-1
+ local pod=liveness-rs0-1
+ set +o xtrace
waiting for pod/liveness-rs0-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HZa6kDD3HE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AXRFycfv6F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HZa6kDD3HE
++ cat /tmp/tmp.AXRFycfv6F
++ rm /tmp/tmp.HZa6kDD3HE /tmp/tmp.AXRFycfv6F
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod liveness-rs0-2
+ local pod=liveness-rs0-2
+ set +o xtrace
waiting for pod/liveness-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z9DDbThqnI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YS0KF654se
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z9DDbThqnI
++ cat /tmp/tmp.YS0KF654se
++ rm /tmp/tmp.z9DDbThqnI /tmp/tmp.YS0KF654se
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KYnBINy8LY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ikIJ14x1TW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb liveness -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KYnBINy8LY
++ cat /tmp/tmp.ikIJ14x1TW
++ rm /tmp/tmp.KYnBINy8LY /tmp/tmp.ikIJ14x1TW
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................................................................................................................................................................................................................................................