+++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/../logs/multi-cluster-service.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/../logs/multi-cluster-service.log
++ arch_scheduling_enabled
++ [[ -n '' ]]
+ [[ 1 -ne 1 ]]
+ desc 'Register Kubernetes cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Register Kubernetes cluster
-----------------------------------------------------------------------------------

+ cat
+ add_arch_to_podspecs -
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ cat -
+ kubectl_bin -n default apply -f -
++ mktemp
+ return
+ local LAST_OUT=/tmp/tmp.JqsxqPtmAm
++ mktemp
+ local LAST_ERR=/tmp/tmp.dkR2nCXWje
+ local exit_status=0
+ local timeout=4
+ args=('-n' 'default' 'apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl -n default apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JqsxqPtmAm
pod/curl created
+ cat /tmp/tmp.dkR2nCXWje
+ rm /tmp/tmp.JqsxqPtmAm /tmp/tmp.dkR2nCXWje
+ return 0
+ kubectl -n default wait --for=condition=Ready pod/curl
pod/curl condition met
+ log 'pod/curl is ready'
+ set +o xtrace
[2026-07-22T14:45:24+0000] pod/curl is ready
++ kubectl exec -n default -it curl -- curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name
Unable to use a TTY - input is not a terminal or the right kind of file
+ k8s_cluster_name=jen-psmdb-2474-67718aa29-2-cluster14
++ kubectl exec -n default -it curl -- curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location
Unable to use a TTY - input is not a terminal or the right kind of file
+ k8s_cluster_region=us-central1-a
+ log gke-cluster=us-central1-a/jen-psmdb-2474-67718aa29-2-cluster14
+ set +o xtrace
[2026-07-22T14:45:27+0000] gke-cluster=us-central1-a/jen-psmdb-2474-67718aa29-2-cluster14
+ gcloud container hub memberships register jen-psmdb-2474-67718aa29-2-cluster14 --gke-cluster us-central1-a/jen-psmdb-2474-67718aa29-2-cluster14 --enable-workload-identity
Membership [projects/cloud-dev-112233/locations/us-central1/memberships/jen-psmdb-2474-67718aa29-2-cluster14] already registered with the cluster [//container.googleapis.com/projects/cloud-dev-112233/locations/us-central1-a/clusters/jen-psmdb-2474-67718aa29-2-cluster14] in the Fleet.
Finished registering to the Fleet.
+ log 'GKE cluster us-central1-a/jen-psmdb-2474-67718aa29-2-cluster14 is registered to the hub'
+ set +o xtrace
[2026-07-22T14:45:31+0000] GKE cluster us-central1-a/jen-psmdb-2474-67718aa29-2-cluster14 is registered to the hub
+ kubectl -n default delete pod curl
pod "curl" deleted from default namespace
+ wait_mcs_api
+ set +o xtrace
waiting for MCS APIs are registered
+ create_infra multi-cluster-service-6584
+ local ns=multi-cluster-service-6584
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.1XkGQPfudx
++ mktemp
+ local LAST_ERR=/tmp/tmp.daj2VGNiXX
+ local exit_status=0
+ local timeout=4
+ args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml' '--ignore-not-found' '--wait=false')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1XkGQPfudx
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbclustersyncs.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.daj2VGNiXX
+ rm /tmp/tmp.1XkGQPfudx /tmp/tmp.daj2VGNiXX
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gJMTopTVK4
++ mktemp
+ local LAST_ERR=/tmp/tmp.BnW3Htoupd
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbbackups.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gJMTopTVK4
+ cat /tmp/tmp.BnW3Htoupd
+ rm /tmp/tmp.gJMTopTVK4 /tmp/tmp.BnW3Htoupd
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbclustersyncs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ kubectl patch perconaservermongodbclustersyncs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbclustersyncs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZMmrhAbTrD
++ mktemp
+ local LAST_ERR=/tmp/tmp.5tchH90g97
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbclustersyncs.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbclustersyncs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZMmrhAbTrD
+ cat /tmp/tmp.5tchH90g97
+ rm /tmp/tmp.ZMmrhAbTrD /tmp/tmp.5tchH90g97
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.RDRmx19jF9
++ mktemp
+ local LAST_ERR=/tmp/tmp.RQmcmPytAs
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbrestores.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RDRmx19jF9
+ cat /tmp/tmp.RQmcmPytAs
+ rm /tmp/tmp.RDRmx19jF9 /tmp/tmp.RQmcmPytAs
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.zbCDwbMxFL
++ mktemp
+ local LAST_ERR=/tmp/tmp.U6cczeLSXy
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'crd' 'perconaservermongodbs.psmdb.percona.com')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zbCDwbMxFL
+ cat /tmp/tmp.U6cczeLSXy
+ rm /tmp/tmp.zbCDwbMxFL /tmp/tmp.U6cczeLSXy
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.BQVwiBuYN2
++ mktemp
+ local LAST_ERR=/tmp/tmp.BwUEjKxxzS
+ local exit_status=0
+ local timeout=4
+ args=('delete' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BQVwiBuYN2
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.BwUEjKxxzS
+ rm /tmp/tmp.BQVwiBuYN2 /tmp/tmp.BwUEjKxxzS
+ return 0
+ check_crd_for_deletion PR-2474-67718aa29
+ local git_tag=PR-2474-67718aa29
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2474-67718aa29/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bPjJzPf27V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uz2T7rKZYB
++ local exit_status=0
++ local timeout=4
++ args=('get' 'crd/null' '-o' 'jsonpath={.status.conditions[-1].type}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bPjJzPf27V
++ cat /tmp/tmp.uz2T7rKZYB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bPjJzPf27V
++ cat /tmp/tmp.uz2T7rKZYB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.bPjJzPf27V
++ cat /tmp/tmp.uz2T7rKZYB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.bPjJzPf27V
++ cat /tmp/tmp.uz2T7rKZYB
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.bPjJzPf27V /tmp/tmp.uz2T7rKZYB
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrolebinding
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ [[ -n '' ]]
+ awk '{print$1}'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.BDFB1dEIAB
++ mktemp
+ local LAST_OUT=/tmp/tmp.T1z5bhnQCs
++ mktemp
+ local LAST_ERR=/tmp/tmp.qexWGXGoDd
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'namespace' 'psmdb-operator' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.gjCy32ak5m
+ local exit_status=0
+ local timeout=4
+ args=('get' 'ns')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BDFB1dEIAB
+ cat /tmp/tmp.gjCy32ak5m
+ rm /tmp/tmp.BDFB1dEIAB /tmp/tmp.gjCy32ak5m
+ return 0
namespace "multi-cluster-service-14330" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T1z5bhnQCs
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.qexWGXGoDd
+ rm /tmp/tmp.T1z5bhnQCs /tmp/tmp.qexWGXGoDd
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.HEmYaCoHVI
++ mktemp
+ local LAST_ERR=/tmp/tmp.MVNlDK2d6d
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'namespace' 'psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HEmYaCoHVI
+ cat /tmp/tmp.MVNlDK2d6d
+ rm /tmp/tmp.HEmYaCoHVI /tmp/tmp.MVNlDK2d6d
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.BKqq9KhHxo
++ mktemp
+ local LAST_ERR=/tmp/tmp.RMRTkrQTGe
+ local exit_status=0
+ local timeout=4
+ args=('create' 'namespace' 'psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BKqq9KhHxo
namespace/psmdb-operator created
+ cat /tmp/tmp.RMRTkrQTGe
+ rm /tmp/tmp.BKqq9KhHxo /tmp/tmp.RMRTkrQTGe
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8G5ibNmsHR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7MojPWgPpp
++ local exit_status=0
++ local timeout=4
++ args=('config' 'current-context')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8G5ibNmsHR
++ cat /tmp/tmp.7MojPWgPpp
++ rm /tmp/tmp.8G5ibNmsHR /tmp/tmp.7MojPWgPpp
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.KFUblzTdne
++ mktemp
+ local LAST_ERR=/tmp/tmp.0o8sY497AW
+ local exit_status=0
+ local timeout=4
+ args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14' '--namespace=psmdb-operator')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KFUblzTdne
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14" modified.
+ cat /tmp/tmp.0o8sY497AW
+ rm /tmp/tmp.KFUblzTdne /tmp/tmp.0o8sY497AW
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.r5ScBH5905
++ mktemp
+ local LAST_ERR=/tmp/tmp.qvTNqUBKm3
+ local exit_status=0
+ local timeout=4
+ args=('apply' '--server-side' '--force-conflicts' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r5ScBH5905
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbclustersyncs.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.qvTNqUBKm3
+ rm /tmp/tmp.r5ScBH5905 /tmp/tmp.qvTNqUBKm3
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.4Mf3kny4W7
++ mktemp
+ local LAST_ERR=/tmp/tmp.NpZW9bLLkv
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-n' 'psmdb-operator' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4Mf3kny4W7
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.NpZW9bLLkv
+ rm /tmp/tmp.4Mf3kny4W7 /tmp/tmp.NpZW9bLLkv
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/deploy/cw-operator.yaml
+ add_arch_to_podspecs -
+ kubectl_bin apply -n psmdb-operator -f -
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ cat -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2Nx4eFk3Ia
++ mktemp
+ local LAST_ERR=/tmp/tmp.hp2kW6lmex
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-n' 'psmdb-operator' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ return
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2Nx4eFk3Ia
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.hp2kW6lmex
+ rm /tmp/tmp.2Nx4eFk3Ia /tmp/tmp.hp2kW6lmex
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vPLzGNi124
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XB49vv6ZgY
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vPLzGNi124
++ cat /tmp/tmp.XB49vv6ZgY
++ rm /tmp/tmp.vPLzGNi124 /tmp/tmp.XB49vv6ZgY
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-555df8f55f-ttjlg
+ local pod=percona-server-mongodb-operator-555df8f55f-ttjlg
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-555df8f55f-ttjlg to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tcBjKRKVzl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0SeRfvoYkx
++ local exit_status=0
++ local timeout=4
++ args=('get' 'pods' '--selector=name=percona-server-mongodb-operator' '-o' 'jsonpath={.items[].metadata.name}' '-n' 'psmdb-operator')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tcBjKRKVzl
++ cat /tmp/tmp.0SeRfvoYkx
++ rm /tmp/tmp.tcBjKRKVzl /tmp/tmp.0SeRfvoYkx
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-555df8f55f-ttjlg
++ mktemp
+ local LAST_OUT=/tmp/tmp.V17G7wz216
++ mktemp
+ local LAST_ERR=/tmp/tmp.q4UGXyQlMC
+ local exit_status=0
+ local timeout=4
+ args=('logs' '-n' 'psmdb-operator' 'percona-server-mongodb-operator-555df8f55f-ttjlg')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-555df8f55f-ttjlg
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V17G7wz216
+ cat /tmp/tmp.q4UGXyQlMC
+ rm /tmp/tmp.V17G7wz216 /tmp/tmp.q4UGXyQlMC
+ return 0
2026-07-22T14:46:44.640Z	INFO	setup	Manager starting up	{"gitCommit": "67718aa293399ea519feb6abcdfab18192ca7f41", "gitBranch": "PR-2474-67718aa29", "buildTime": "", "goVersion": "go1.26.5", "os": "linux", "arch": "amd64"}
+ create_namespace multi-cluster-service-6584
+ local namespace=multi-cluster-service-6584
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
++ mktemp
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.RBcmbrx32W
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns
+ [[ -n '' ]]
+ desc 'cleaned up old namespaces multi-cluster-service-6584'
+ local LAST_ERR=/tmp/tmp.cNhuDLEgqz
+ local exit_status=0
+ local timeout=4
+ args=('get' 'ns')
+ local args
+ transformed_files=()
+ set +o xtrace
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]


-----------------------------------------------------------------------------------
cleaned up old namespaces multi-cluster-service-6584
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace multi-cluster-service-6584 --ignore-not-found
++ seq 0 2
++ mktemp
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_OUT=/tmp/tmp.vX7r5tjuxI
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZCezRxodS3
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'namespace' 'multi-cluster-service-6584' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace multi-cluster-service-6584 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RBcmbrx32W
+ cat /tmp/tmp.cNhuDLEgqz
+ rm /tmp/tmp.RBcmbrx32W /tmp/tmp.cNhuDLEgqz
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vX7r5tjuxI
+ cat /tmp/tmp.ZCezRxodS3
+ rm /tmp/tmp.vX7r5tjuxI /tmp/tmp.ZCezRxodS3
+ return 0
+ kubectl_bin wait --for=delete namespace multi-cluster-service-6584
++ mktemp
+ local LAST_OUT=/tmp/tmp.vJcTS4KfWV
++ mktemp
+ local LAST_ERR=/tmp/tmp.5fYuNHA1mJ
+ local exit_status=0
+ local timeout=4
+ args=('wait' '--for=delete' 'namespace' 'multi-cluster-service-6584')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace multi-cluster-service-6584
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vJcTS4KfWV
+ cat /tmp/tmp.5fYuNHA1mJ
+ rm /tmp/tmp.vJcTS4KfWV /tmp/tmp.5fYuNHA1mJ
+ return 0
+ desc 'create namespace multi-cluster-service-6584'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace multi-cluster-service-6584
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace multi-cluster-service-6584
++ mktemp
+ local LAST_OUT=/tmp/tmp.iWXTW0zBO1
++ mktemp
+ local LAST_ERR=/tmp/tmp.yflJvoH1Eg
+ local exit_status=0
+ local timeout=4
+ args=('create' 'namespace' 'multi-cluster-service-6584')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace multi-cluster-service-6584
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iWXTW0zBO1
namespace/multi-cluster-service-6584 created
+ cat /tmp/tmp.yflJvoH1Eg
+ rm /tmp/tmp.iWXTW0zBO1 /tmp/tmp.yflJvoH1Eg
+ return 0
+ set_kube_ctx multi-cluster-service-6584
+ local namespace=multi-cluster-service-6584
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j874BQ4nPp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BG0IZJOfEZ
++ local exit_status=0
++ local timeout=4
++ args=('config' 'current-context')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.j874BQ4nPp
++ cat /tmp/tmp.BG0IZJOfEZ
++ rm /tmp/tmp.j874BQ4nPp /tmp/tmp.BG0IZJOfEZ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14 --namespace=multi-cluster-service-6584
++ mktemp
+ local LAST_OUT=/tmp/tmp.bvV60NfxdC
++ mktemp
+ local LAST_ERR=/tmp/tmp.GxDt9Mwliu
+ local exit_status=0
+ local timeout=4
+ args=('config' 'set-context' 'gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14' '--namespace=multi-cluster-service-6584')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14 --namespace=multi-cluster-service-6584
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bvV60NfxdC
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2474-67718aa29-2-cluster14" modified.
+ cat /tmp/tmp.GxDt9Mwliu
+ rm /tmp/tmp.bvV60NfxdC /tmp/tmp.GxDt9Mwliu
+ return 0
+ desc 'create first PSMDB cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yL8CpgAhdC
++ mktemp
+ local LAST_ERR=/tmp/tmp.itXlbD3DHd
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/secrets.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yL8CpgAhdC
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.itXlbD3DHd
+ rm /tmp/tmp.yL8CpgAhdC /tmp/tmp.itXlbD3DHd
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.h49EQobacV
++ mktemp
+ local LAST_ERR=/tmp/tmp.SV4Y23NpPs
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml' '-f' '/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h49EQobacV
secret/minio-secret created
secret/aws-s3-secret created
secret/alibaba-cloud-secret created
secret/oci-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.SV4Y23NpPs
+ rm /tmp/tmp.h49EQobacV /tmp/tmp.SV4Y23NpPs
+ return 0
+ create_runtime_class
+ kubectl_bin delete runtimeclass container-rc --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.fzIX0fMNG8
++ mktemp
+ local LAST_ERR=/tmp/tmp.RgYo8giTga
+ local exit_status=0
+ local timeout=4
+ args=('delete' 'runtimeclass' 'container-rc' '--ignore-not-found')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete runtimeclass container-rc --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fzIX0fMNG8
runtimeclass.node.k8s.io "container-rc" deleted
+ cat /tmp/tmp.RgYo8giTga
+ rm /tmp/tmp.fzIX0fMNG8 /tmp/tmp.RgYo8giTga
+ return 0
+ local handler=runc
+ desc 'Create RuntimeClass with handler runc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Create RuntimeClass with handler runc
-----------------------------------------------------------------------------------

+ yq eval '.handler = "runc"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/conf/container-rc.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.JjsojwPcNM
++ mktemp
+ local LAST_ERR=/tmp/tmp.MJd511lEtD
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JjsojwPcNM
runtimeclass.node.k8s.io/container-rc created
+ cat /tmp/tmp.MJd511lEtD
+ rm /tmp/tmp.JjsojwPcNM /tmp/tmp.MJd511lEtD
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/conf/some-name.yml
+ kubectl_bin apply -f -
++ mktemp
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2474/e2e-tests/multi-cluster-service/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ local LAST_OUT=/tmp/tmp.4ecrhORW9Y
+ [[ -n docker.io/perconalab/fluentbit:main-logcollector ]]
+ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector
+ yq eval '(.spec | select(has("search"))).search.image = "perconalab/percona-server-mongodb-operator:main-mongot"'
+ add_arch_to_psmdb -
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2474-67718aa29"'
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ cat -
++ mktemp
+ yq eval '(.spec | select(has("logcollector"))).logcollector.image = strenv(IMAGE_LOGCOLLECTOR)'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/multi-cluster-service-6584/g
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_ERR=/tmp/tmp.6xvxXKL0YW
+ local exit_status=0
+ local timeout=4
+ args=('apply' '-f' '-')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ return
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4ecrhORW9Y
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.6xvxXKL0YW
+ rm /tmp/tmp.4ecrhORW9Y /tmp/tmp.6xvxXKL0YW
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..............OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aVi2D7DjYD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lwteb0omaL
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aVi2D7DjYD
++ cat /tmp/tmp.lwteb0omaL
++ rm /tmp/tmp.aVi2D7DjYD /tmp/tmp.lwteb0omaL
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.............OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.crsuH1uIwe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2KWt43kwGi
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.crsuH1uIwe
++ cat /tmp/tmp.2KWt43kwGi
++ rm /tmp/tmp.crsuH1uIwe /tmp/tmp.2KWt43kwGi
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M1EhRW7Zib
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4jlqVgl3dl
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M1EhRW7Zib
++ cat /tmp/tmp.4jlqVgl3dl
++ rm /tmp/tmp.M1EhRW7Zib /tmp/tmp.4jlqVgl3dl
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster some-name to be ready.....................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readiness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iqIYPm3ToD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RZeRiH9lLp
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iqIYPm3ToD
++ cat /tmp/tmp.RZeRiH9lLp
++ rm /tmp/tmp.iqIYPm3ToD /tmp/tmp.RZeRiH9lLp
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hPkg5NKocF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZdcTMKsXeH
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hPkg5NKocF
++ cat /tmp/tmp.ZdcTMKsXeH
++ rm /tmp/tmp.hPkg5NKocF /tmp/tmp.ZdcTMKsXeH
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eT7zRL8lwL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z1TtgYfotn
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eT7zRL8lwL
++ cat /tmp/tmp.Z1TtgYfotn
++ rm /tmp/tmp.eT7zRL8lwL /tmp/tmp.Z1TtgYfotn
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readiness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oqq2ouYBHt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ct1Hl4pIeu
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Oqq2ouYBHt
++ cat /tmp/tmp.Ct1Hl4pIeu
++ rm /tmp/tmp.Oqq2ouYBHt /tmp/tmp.Ct1Hl4pIeu
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ErJuE5gxvw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Rd5gP3GKw
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ErJuE5gxvw
++ cat /tmp/tmp.9Rd5gP3GKw
++ rm /tmp/tmp.ErJuE5gxvw /tmp/tmp.9Rd5gP3GKw
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iIHeMwV4B6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EB7WGseIgo
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iIHeMwV4B6
++ cat /tmp/tmp.EB7WGseIgo
++ rm /tmp/tmp.iIHeMwV4B6 /tmp/tmp.EB7WGseIgo
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster some-name to be ready
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R96AU09QaQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qMFNIMSA11
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R96AU09QaQ
++ cat /tmp/tmp.qMFNIMSA11
++ rm /tmp/tmp.R96AU09QaQ /tmp/tmp.qMFNIMSA11
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ desc 'enable MCS'
+ set +o xtrace


-----------------------------------------------------------------------------------
enable MCS
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type json '-p=[{"op":"add","path":"/spec/multiCluster/enabled","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.3u3fpSug5U
++ mktemp
+ local LAST_ERR=/tmp/tmp.A1WkjpGEqb
+ local exit_status=0
+ local timeout=4
+ args=('patch' 'psmdb' 'some-name' '--type' 'json' '-p=[{"op":"add","path":"/spec/multiCluster/enabled","value":true}]')
+ local args
+ transformed_files=()
+ local transformed_files
+ arch_scheduling_enabled
+ [[ -n '' ]]
+ arch_scheduling_enabled
+ [[ -n '' ]]
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type json '-p=[{"op":"add","path":"/spec/multiCluster/enabled","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3u3fpSug5U
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.A1WkjpGEqb
+ rm /tmp/tmp.3u3fpSug5U /tmp/tmp.A1WkjpGEqb
+ return 0
+ desc 'check if ServiceExport objects are created'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if ServiceExport objects are created
-----------------------------------------------------------------------------------

+ wait_service_export
+ set +o xtrace
waiting for cluster services to be exported
+ wait_service_import
+ set +o xtrace
waiting for cluster services to be imported..........................................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster some-name to be ready'
waiting for cluster some-name to be ready++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hUG4y0dbsq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cvHRafNbEy
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hUG4y0dbsq
++ cat /tmp/tmp.cvHRafNbEy
++ rm /tmp/tmp.hUG4y0dbsq /tmp/tmp.cvHRafNbEy
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vnx20zAqdX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Wky8L0xP53
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vnx20zAqdX
++ cat /tmp/tmp.Wky8L0xP53
++ rm /tmp/tmp.vnx20zAqdX /tmp/tmp.Wky8L0xP53
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xo1JWpvAgq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.noU2y9G4BN
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xo1JWpvAgq
++ cat /tmp/tmp.noU2y9G4BN
++ rm /tmp/tmp.xo1JWpvAgq /tmp/tmp.noU2y9G4BN
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4LPa4Ec5Yl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fatLnPBBvr
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4LPa4Ec5Yl
++ cat /tmp/tmp.fatLnPBBvr
++ rm /tmp/tmp.4LPa4Ec5Yl /tmp/tmp.fatLnPBBvr
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o3d5DZDk11
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YIa2j1lhW0
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.o3d5DZDk11
++ cat /tmp/tmp.YIa2j1lhW0
++ rm /tmp/tmp.o3d5DZDk11 /tmp/tmp.YIa2j1lhW0
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ohSY6edW8P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AIKAS7hx1d
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ohSY6edW8P
++ cat /tmp/tmp.AIKAS7hx1d
++ rm /tmp/tmp.ohSY6edW8P /tmp/tmp.AIKAS7hx1d
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HvsFMYsmE7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f37gEAAWtv
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HvsFMYsmE7
++ cat /tmp/tmp.f37gEAAWtv
++ rm /tmp/tmp.HvsFMYsmE7 /tmp/tmp.f37gEAAWtv
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iXX30dixln
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NH72ipmu7M
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iXX30dixln
++ cat /tmp/tmp.NH72ipmu7M
++ rm /tmp/tmp.iXX30dixln /tmp/tmp.NH72ipmu7M
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.INQWIz4lVS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OA3SgxZZAK
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.INQWIz4lVS
++ cat /tmp/tmp.OA3SgxZZAK
++ rm /tmp/tmp.INQWIz4lVS /tmp/tmp.OA3SgxZZAK
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PRnKvXWEvw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DlWUs2WrOC
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PRnKvXWEvw
++ cat /tmp/tmp.DlWUs2WrOC
++ rm /tmp/tmp.PRnKvXWEvw /tmp/tmp.DlWUs2WrOC
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ypP1H6pHFH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RsvN35aLKk
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ypP1H6pHFH
++ cat /tmp/tmp.RsvN35aLKk
++ rm /tmp/tmp.ypP1H6pHFH /tmp/tmp.RsvN35aLKk
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yzg5lqQEn9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3g85CNlr5c
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yzg5lqQEn9
++ cat /tmp/tmp.3g85CNlr5c
++ rm /tmp/tmp.yzg5lqQEn9 /tmp/tmp.3g85CNlr5c
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CAWbrUGLgA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EGfxHcwzwf
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CAWbrUGLgA
++ cat /tmp/tmp.EGfxHcwzwf
++ rm /tmp/tmp.CAWbrUGLgA /tmp/tmp.EGfxHcwzwf
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OXUbaVvaZy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bwkUTtsxWt
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OXUbaVvaZy
++ cat /tmp/tmp.bwkUTtsxWt
++ rm /tmp/tmp.OXUbaVvaZy /tmp/tmp.bwkUTtsxWt
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 14 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sDt1f5TSYW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aI228Y6XZZ
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sDt1f5TSYW
++ cat /tmp/tmp.aI228Y6XZZ
++ rm /tmp/tmp.sDt1f5TSYW /tmp/tmp.aI228Y6XZZ
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 15 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IW3SvvqjSL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HwQkFIVmoX
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IW3SvvqjSL
++ cat /tmp/tmp.HwQkFIVmoX
++ rm /tmp/tmp.IW3SvvqjSL /tmp/tmp.HwQkFIVmoX
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 16 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UxR9PQXg2z
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vCkTwGkQTQ
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UxR9PQXg2z
++ cat /tmp/tmp.vCkTwGkQTQ
++ rm /tmp/tmp.UxR9PQXg2z /tmp/tmp.vCkTwGkQTQ
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 17 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.djgaMVGlEt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ipvVjaLSoB
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.djgaMVGlEt
++ cat /tmp/tmp.ipvVjaLSoB
++ rm /tmp/tmp.djgaMVGlEt /tmp/tmp.ipvVjaLSoB
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 18 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y4MaYUIFCK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OQMQ13VD1t
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y4MaYUIFCK
++ cat /tmp/tmp.OQMQ13VD1t
++ rm /tmp/tmp.y4MaYUIFCK /tmp/tmp.OQMQ13VD1t
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 19 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xGkjJDiyni
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m9NSv2KLMN
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xGkjJDiyni
++ cat /tmp/tmp.m9NSv2KLMN
++ rm /tmp/tmp.xGkjJDiyni /tmp/tmp.m9NSv2KLMN
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 20 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1eCFTByCT0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.R2dLAJnm3n
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1eCFTByCT0
++ cat /tmp/tmp.R2dLAJnm3n
++ rm /tmp/tmp.1eCFTByCT0 /tmp/tmp.R2dLAJnm3n
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 21 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F3xktDWcU3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qQ8TJmcmgT
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F3xktDWcU3
++ cat /tmp/tmp.qQ8TJmcmgT
++ rm /tmp/tmp.F3xktDWcU3 /tmp/tmp.qQ8TJmcmgT
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 22 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BvVshmcaEM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OyErqOzOX3
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BvVshmcaEM
++ cat /tmp/tmp.OyErqOzOX3
++ rm /tmp/tmp.BvVshmcaEM /tmp/tmp.OyErqOzOX3
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 23 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SAdyH9u9UN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AarylslEpq
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SAdyH9u9UN
++ cat /tmp/tmp.AarylslEpq
++ rm /tmp/tmp.SAdyH9u9UN /tmp/tmp.AarylslEpq
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 24 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xCfAjOlNjn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lKJVVDakny
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xCfAjOlNjn
++ cat /tmp/tmp.lKJVVDakny
++ rm /tmp/tmp.xCfAjOlNjn /tmp/tmp.lKJVVDakny
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 25 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.y8SRsmtC92
+++ mktemp
++ local LAST_ERR=/tmp/tmp.I5fLxHrXf2
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.y8SRsmtC92
++ cat /tmp/tmp.I5fLxHrXf2
++ rm /tmp/tmp.y8SRsmtC92 /tmp/tmp.I5fLxHrXf2
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 26 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DycKU0Uib7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0BzIqtjDbj
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DycKU0Uib7
++ cat /tmp/tmp.0BzIqtjDbj
++ rm /tmp/tmp.DycKU0Uib7 /tmp/tmp.0BzIqtjDbj
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 27 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IpnOMufbIk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6F9Pzlsuq6
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IpnOMufbIk
++ cat /tmp/tmp.6F9Pzlsuq6
++ rm /tmp/tmp.IpnOMufbIk /tmp/tmp.6F9Pzlsuq6
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 28 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZvDbTOXZAn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yyam16YvVX
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZvDbTOXZAn
++ cat /tmp/tmp.Yyam16YvVX
++ rm /tmp/tmp.ZvDbTOXZAn /tmp/tmp.Yyam16YvVX
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 29 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gpdEd805MP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eGqysios39
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gpdEd805MP
++ cat /tmp/tmp.eGqysios39
++ rm /tmp/tmp.gpdEd805MP /tmp/tmp.eGqysios39
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 30 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rkkwIN10YI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m9yMPQ7DVA
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rkkwIN10YI
++ cat /tmp/tmp.m9yMPQ7DVA
++ rm /tmp/tmp.rkkwIN10YI /tmp/tmp.m9yMPQ7DVA
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 31 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kff600GQis
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BOylboj7tT
++ local exit_status=0
++ local timeout=4
++ args=('get' 'psmdb' 'some-name' '-o' 'jsonpath={.status.state}')
++ local args
++ transformed_files=()
++ local transformed_files
++ arch_scheduling_enabled
++ [[ -n '' ]]
++ arch_scheduling_enabled
++ [[ -n '' ]]
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Kff600GQis
++ cat /tmp/tmp.BOylboj7tT
++ rm /tmp/tmp.Kff600GQis /tmp/tmp.BOylboj7tT
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 32 -ge 32 ']'
+ echo max retry count 32 reached. something went wrong with operator or kubernetes cluster
max retry count 32 reached. something went wrong with operator or kubernetes cluster
+ exit 1