++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/multi-cluster-service.log'
Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/multi-cluster-service.log
++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP_AZURE=
++ oc get projects
++ kubectl get nodes
++ grep '^minikube'
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep -eks-
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' ']'
++ EKS=0
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep gke
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ '[' v1.33.11-gke.1074000 ']'
++ GKE=1
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/sbin/sed -r 's/[^0-9.]+//g'
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.33
+ '[' 1 -ne 1 ']'
+ desc 'Register Kubernetes cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Register Kubernetes cluster
-----------------------------------------------------------------------------------

+ cat
+ kubectl -n default apply -f -
pod/curl created
+ kubectl -n default wait --for=condition=Ready pod/curl
pod/curl condition met
+ log 'pod/curl is ready'
+ set +o xtrace
[2026-06-09T09:13:45+0000] pod/curl is ready
++ kubectl exec -n default -it curl -- curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name
Unable to use a TTY - input is not a terminal or the right kind of file
+ k8s_cluster_name=jen-psmdb-2058-fb8cc7187-17-cluster10
++ kubectl exec -n default -it curl -- curl -s -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location
Unable to use a TTY - input is not a terminal or the right kind of file
+ k8s_cluster_region=us-central1-a
+ log gke-cluster=us-central1-a/jen-psmdb-2058-fb8cc7187-17-cluster10
+ set +o xtrace
[2026-06-09T09:13:48+0000] gke-cluster=us-central1-a/jen-psmdb-2058-fb8cc7187-17-cluster10
+ gcloud container hub memberships register jen-psmdb-2058-fb8cc7187-17-cluster10 --gke-cluster us-central1-a/jen-psmdb-2058-fb8cc7187-17-cluster10 --enable-workload-identity
Waiting for membership to be created...
...........................................done.
Finished registering to the Fleet.
+ log 'GKE cluster us-central1-a/jen-psmdb-2058-fb8cc7187-17-cluster10 is registered to the hub'
+ set +o xtrace
[2026-06-09T09:14:01+0000] GKE cluster us-central1-a/jen-psmdb-2058-fb8cc7187-17-cluster10 is registered to the hub
+ kubectl -n default delete pod curl
pod "curl" deleted from default namespace
+ wait_mcs_api
+ set +o xtrace
waiting for MCS APIs are registered.
+ create_infra multi-cluster-service-9794
+ local ns=multi-cluster-service-9794
+ echo multi-cluster-service-9794
+ [[ 0 == 1 ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ xargs kubectl delete ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
++ mktemp
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.47lMWNUK6Q
+ local LAST_OUT=/tmp/tmp.hCVGQa5w8r
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.mP9PO7LaEy
+ local LAST_ERR=/tmp/tmp.MbS8yiiobs
+ local exit_status=0
+ local exit_status=0
+ local timeout=4
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hCVGQa5w8r
+ cat /tmp/tmp.mP9PO7LaEy
+ rm /tmp/tmp.hCVGQa5w8r /tmp/tmp.mP9PO7LaEy
+ return 0
namespace "demand-backup-physical-sharded-minio-14307" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.47lMWNUK6Q
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.MbS8yiiobs
+ rm /tmp/tmp.47lMWNUK6Q /tmp/tmp.MbS8yiiobs
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.5K9OmcuA7g
++ mktemp
+ local LAST_ERR=/tmp/tmp.IvurpnCjwF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5K9OmcuA7g
+ cat /tmp/tmp.IvurpnCjwF
+ rm /tmp/tmp.5K9OmcuA7g /tmp/tmp.IvurpnCjwF
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.0a7Vgx0zsA
++ mktemp
+ local LAST_ERR=/tmp/tmp.7EiFgmTuI9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0a7Vgx0zsA
namespace/psmdb-operator created
+ cat /tmp/tmp.7EiFgmTuI9
+ rm /tmp/tmp.0a7Vgx0zsA /tmp/tmp.7EiFgmTuI9
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qZIzemXmkJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.db2QgSvhir
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qZIzemXmkJ
++ cat /tmp/tmp.db2QgSvhir
++ rm /tmp/tmp.qZIzemXmkJ /tmp/tmp.db2QgSvhir
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YVB9sJMxO1
++ mktemp
+ local LAST_ERR=/tmp/tmp.l7bQkcOBRd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YVB9sJMxO1
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10" modified.
+ cat /tmp/tmp.l7bQkcOBRd
+ rm /tmp/tmp.YVB9sJMxO1 /tmp/tmp.l7bQkcOBRd
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/multi-cluster-service/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.0ytaSCeAmX
++ mktemp
+ local LAST_ERR=/tmp/tmp.7p6ZoUCPWh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0ytaSCeAmX
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.7p6ZoUCPWh
+ rm /tmp/tmp.0ytaSCeAmX /tmp/tmp.7p6ZoUCPWh
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Z2un7WLZqw
++ mktemp
+ local LAST_ERR=/tmp/tmp.B8sFvnH9LM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Z2un7WLZqw
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged
+ cat /tmp/tmp.B8sFvnH9LM
+ rm /tmp/tmp.Z2un7WLZqw /tmp/tmp.B8sFvnH9LM
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.xSZqV6M7nS
++ mktemp
+ local LAST_ERR=/tmp/tmp.bjbAYYy01j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xSZqV6M7nS
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.bjbAYYy01j
+ rm /tmp/tmp.xSZqV6M7nS /tmp/tmp.bjbAYYy01j
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FK7X4HIzLs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zM3yCAzXdX
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FK7X4HIzLs
++ cat /tmp/tmp.zM3yCAzXdX
++ rm /tmp/tmp.FK7X4HIzLs /tmp/tmp.zM3yCAzXdX
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-jfmrm
+ local pod=percona-server-mongodb-operator-859f4bc6d7-jfmrm
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-859f4bc6d7-jfmrm to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mR9nWovSFq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.plspXuEWUB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mR9nWovSFq
++ cat /tmp/tmp.plspXuEWUB
++ rm /tmp/tmp.mR9nWovSFq /tmp/tmp.plspXuEWUB
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-jfmrm
++ mktemp
+ local LAST_OUT=/tmp/tmp.FgFpWNfGTL
++ mktemp
+ local LAST_ERR=/tmp/tmp.D6UdKrKVHF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-jfmrm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FgFpWNfGTL
+ cat /tmp/tmp.D6UdKrKVHF
+ rm /tmp/tmp.FgFpWNfGTL /tmp/tmp.D6UdKrKVHF
+ return 0
2026-06-09T09:14:58.113Z	INFO	setup	Manager starting up	{"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"}
+ create_namespace multi-cluster-service-9794
+ local namespace=multi-cluster-service-9794
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
++ mktemp
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces multi-cluster-service-9794'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces multi-cluster-service-9794
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace multi-cluster-service-9794 --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.OhcpojuGtK
++ mktemp
+ local LAST_OUT=/tmp/tmp.sZUygsBttr
++ mktemp
+ local LAST_ERR=/tmp/tmp.1sj7tqnQXy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.pPSzb9C1iF
+ local exit_status=0
+ local timeout=4
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace multi-cluster-service-9794 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OhcpojuGtK
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sZUygsBttr
+ cat /tmp/tmp.1sj7tqnQXy
+ rm /tmp/tmp.OhcpojuGtK /tmp/tmp.1sj7tqnQXy
+ cat /tmp/tmp.pPSzb9C1iF
+ rm /tmp/tmp.sZUygsBttr /tmp/tmp.pPSzb9C1iF
+ return 0
+ return 0
+ kubectl_bin wait --for=delete namespace multi-cluster-service-9794
++ mktemp
+ local LAST_OUT=/tmp/tmp.vdqmYKhZ59
++ mktemp
+ local LAST_ERR=/tmp/tmp.GLev9hWO0S
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace multi-cluster-service-9794
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vdqmYKhZ59
+ cat /tmp/tmp.GLev9hWO0S
+ rm /tmp/tmp.vdqmYKhZ59 /tmp/tmp.GLev9hWO0S
+ return 0
+ desc 'create namespace multi-cluster-service-9794'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace multi-cluster-service-9794
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace multi-cluster-service-9794
++ mktemp
+ local LAST_OUT=/tmp/tmp.wHo5AEzwaW
++ mktemp
+ local LAST_ERR=/tmp/tmp.u6USGhTatj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace multi-cluster-service-9794
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wHo5AEzwaW
namespace/multi-cluster-service-9794 created
+ cat /tmp/tmp.u6USGhTatj
+ rm /tmp/tmp.wHo5AEzwaW /tmp/tmp.u6USGhTatj
+ return 0
+ set_kube_ctx multi-cluster-service-9794
+ local namespace=multi-cluster-service-9794
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CvBGtF6him
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TqLEgkiASA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CvBGtF6him
++ cat /tmp/tmp.TqLEgkiASA
++ rm /tmp/tmp.CvBGtF6him /tmp/tmp.TqLEgkiASA
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10 --namespace=multi-cluster-service-9794
++ mktemp
+ local LAST_OUT=/tmp/tmp.lLmFzAqpx5
++ mktemp
+ local LAST_ERR=/tmp/tmp.dUbVKC9gZJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10 --namespace=multi-cluster-service-9794
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lLmFzAqpx5
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster10" modified.
+ cat /tmp/tmp.dUbVKC9gZJ
+ rm /tmp/tmp.lLmFzAqpx5 /tmp/tmp.dUbVKC9gZJ
+ return 0
+ desc 'create first PSMDB cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster
-----------------------------------------------------------------------------------

+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.LqG90kgcCV
++ mktemp
+ local LAST_ERR=/tmp/tmp.GjS4MX1mMz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LqG90kgcCV
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.GjS4MX1mMz
+ rm /tmp/tmp.LqG90kgcCV /tmp/tmp.GjS4MX1mMz
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tBLVS57hdP
++ mktemp
+ local LAST_ERR=/tmp/tmp.fvH3fc1eUP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tBLVS57hdP
secret/minio-secret created
secret/aws-s3-secret created
secret/alibaba-cloud-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.fvH3fc1eUP
+ rm /tmp/tmp.tBLVS57hdP /tmp/tmp.fvH3fc1eUP
+ return 0
+ version_gt 1.19
++ echo '1.33 >= 1.19'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' 0 -ne 1 ']'
+ /usr/sbin/sed s/docker/runc/g
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/container-rc.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Oeb2HDyhee
++ mktemp
+ local LAST_ERR=/tmp/tmp.L3fjNbYt8r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Oeb2HDyhee
runtimeclass.node.k8s.io/container-rc created
+ cat /tmp/tmp.L3fjNbYt8r
+ rm /tmp/tmp.Oeb2HDyhee /tmp/tmp.L3fjNbYt8r
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/multi-cluster-service/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/multi-cluster-service/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/multi-cluster-service/conf/some-name.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
+ /usr/sbin/sed -e s/NAME_SPACE/multi-cluster-service-9794/g
+ local LAST_OUT=/tmp/tmp.lp4WfFwCvM
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.7H4BIZ1vCA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lp4WfFwCvM
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.7H4BIZ1vCA
+ rm /tmp/tmp.lp4WfFwCvM /tmp/tmp.7H4BIZ1vCA
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready...........OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.............OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d0MNHNyYAF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5iFcSwfcLQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d0MNHNyYAF
++ cat /tmp/tmp.5iFcSwfcLQ
++ rm /tmp/tmp.d0MNHNyYAF /tmp/tmp.5iFcSwfcLQ
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready................OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CsDKBX4oVe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5V4jOTvm8S
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CsDKBX4oVe
++ cat /tmp/tmp.5V4jOTvm8S
++ rm /tmp/tmp.CsDKBX4oVe /tmp/tmp.5V4jOTvm8S
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lE5zR8nffU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MDigku8ajt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lE5zR8nffU
++ cat /tmp/tmp.MDigku8ajt
++ rm /tmp/tmp.lE5zR8nffU /tmp/tmp.MDigku8ajt
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness.......................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3yf7lZUTG3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S0aQr1eatI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3yf7lZUTG3
++ cat /tmp/tmp.S0aQr1eatI
++ rm /tmp/tmp.3yf7lZUTG3 /tmp/tmp.S0aQr1eatI
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jdT6v3O28D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XNSuQpxdS7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jdT6v3O28D
++ cat /tmp/tmp.XNSuQpxdS7
++ rm /tmp/tmp.jdT6v3O28D /tmp/tmp.XNSuQpxdS7
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4tca5fkVZI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kIY9h3Xc9F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4tca5fkVZI
++ cat /tmp/tmp.kIY9h3Xc9F
++ rm /tmp/tmp.4tca5fkVZI /tmp/tmp.kIY9h3Xc9F
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ false == true ]]
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in $(seq 0 $last_pod)
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C2fZ5cHz5k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UlWNQq5jTZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C2fZ5cHz5k
++ cat /tmp/tmp.UlWNQq5jTZ
++ rm /tmp/tmp.C2fZ5cHz5k /tmp/tmp.UlWNQq5jTZ
++ return 0
+ [[ '' == true ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BKzAavcy1u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5hQol6RfNS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].nonvoting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BKzAavcy1u
++ cat /tmp/tmp.5hQol6RfNS
++ rm /tmp/tmp.BKzAavcy1u /tmp/tmp.5hQol6RfNS
++ return 0
+ [[ '' == true ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3JSWrEbBiH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IkKjZd1JoB
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3JSWrEbBiH
++ cat /tmp/tmp.IkKjZd1JoB
++ rm /tmp/tmp.3JSWrEbBiH /tmp/tmp.IkKjZd1JoB
++ return 0
+ [[ '' == true ]]
+ sleep 10
+ [[ true == true ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JiQNLLrcrq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eFVhQbK499
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JiQNLLrcrq
++ cat /tmp/tmp.eFVhQbK499
++ rm /tmp/tmp.JiQNLLrcrq /tmp/tmp.eFVhQbK499
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ desc 'enable MCS'
+ set +o xtrace


-----------------------------------------------------------------------------------
enable MCS
-----------------------------------------------------------------------------------

+ kubectl_bin patch psmdb some-name --type json '-p=[{"op":"add","path":"/spec/multiCluster/enabled","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.X75GczapEE
++ mktemp
+ local LAST_ERR=/tmp/tmp.y03tSxbEUe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl patch psmdb some-name --type json '-p=[{"op":"add","path":"/spec/multiCluster/enabled","value":true}]'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.X75GczapEE
perconaservermongodb.psmdb.percona.com/some-name patched
+ cat /tmp/tmp.y03tSxbEUe
+ rm /tmp/tmp.X75GczapEE /tmp/tmp.y03tSxbEUe
+ return 0
+ desc 'check if ServiceExport objects are created'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if ServiceExport objects are created
-----------------------------------------------------------------------------------

+ wait_service_export
+ set +o xtrace
waiting for cluster services to be exported
+ wait_service_import
+ set +o xtrace
waiting for cluster services to be imported....................
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.szDH1L1fLV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tWHNQStAlP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.szDH1L1fLV
++ cat /tmp/tmp.tWHNQStAlP
++ rm /tmp/tmp.szDH1L1fLV /tmp/tmp.tWHNQStAlP
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3dgnstIdsK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cTE7AT51Dn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3dgnstIdsK
++ cat /tmp/tmp.cTE7AT51Dn
++ rm /tmp/tmp.3dgnstIdsK /tmp/tmp.cTE7AT51Dn
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hMWhiKoVav
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JNf7b1kx7U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hMWhiKoVav
++ cat /tmp/tmp.JNf7b1kx7U
++ rm /tmp/tmp.hMWhiKoVav /tmp/tmp.JNf7b1kx7U
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E8PsQGQjWS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Kooz34ydeq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.E8PsQGQjWS
++ cat /tmp/tmp.Kooz34ydeq
++ rm /tmp/tmp.E8PsQGQjWS /tmp/tmp.Kooz34ydeq
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bgMftiLC32
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pfsvdP5XtV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bgMftiLC32
++ cat /tmp/tmp.pfsvdP5XtV
++ rm /tmp/tmp.bgMftiLC32 /tmp/tmp.pfsvdP5XtV
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OAIWrjb6Bk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Je1OspUe7d
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OAIWrjb6Bk
++ cat /tmp/tmp.Je1OspUe7d
++ rm /tmp/tmp.OAIWrjb6Bk /tmp/tmp.Je1OspUe7d
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.suKEUhYkLi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pgBXLrtXmS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.suKEUhYkLi
++ cat /tmp/tmp.pgBXLrtXmS
++ rm /tmp/tmp.suKEUhYkLi /tmp/tmp.pgBXLrtXmS
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8ZuDdjj0IO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N5Q9d3Xg5G
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8ZuDdjj0IO
++ cat /tmp/tmp.N5Q9d3Xg5G
++ rm /tmp/tmp.8ZuDdjj0IO /tmp/tmp.N5Q9d3Xg5G
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s8t1XFF4lr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6g8QZTjDeS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s8t1XFF4lr
++ cat /tmp/tmp.6g8QZTjDeS
++ rm /tmp/tmp.s8t1XFF4lr /tmp/tmp.6g8QZTjDeS
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qdYmDBs8wm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lDqngaO0yn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qdYmDBs8wm
++ cat /tmp/tmp.lDqngaO0yn
++ rm /tmp/tmp.qdYmDBs8wm /tmp/tmp.lDqngaO0yn
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.taRejY8deN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7nFQyDUU7w
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.taRejY8deN
++ cat /tmp/tmp.7nFQyDUU7w
++ rm /tmp/tmp.taRejY8deN /tmp/tmp.7nFQyDUU7w
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XZCVuYXBoP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iNUtegER7v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XZCVuYXBoP
++ cat /tmp/tmp.iNUtegER7v
++ rm /tmp/tmp.XZCVuYXBoP /tmp/tmp.iNUtegER7v
++ return 0
+ [[ error == ready ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tEoKp7zmk9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gwRXWIMo0g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tEoKp7zmk9
++ cat /tmp/tmp.gwRXWIMo0g
++ rm /tmp/tmp.tEoKp7zmk9 /tmp/tmp.gwRXWIMo0g
++ return 0
+ [[ initializing == ready ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X1GmYDqnau
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KoYJh7QKAu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X1GmYDqnau
++ cat /tmp/tmp.KoYJh7QKAu
++ rm /tmp/tmp.X1GmYDqnau /tmp/tmp.KoYJh7QKAu
++ return 0
+ [[ ready == ready ]]
+ echo .OK
.OK
+ desc 'delete cluster membership'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete cluster membership
-----------------------------------------------------------------------------------

+ gcloud container fleet memberships delete jen-psmdb-2058-fb8cc7187-17-cluster10 --quiet
Delete request issued for: [jen-psmdb-2058-fb8cc7187-17-cluster10]
Waiting for operation [projects/cloud-dev-112233/locations/us-central1/operations/operation-1780997125010-653ceb4fb7335-2f3dd85e-d7f87522] to complete...
..................................................................................................................................................................................................................................done.
Deleted membership [jen-psmdb-2058-fb8cc7187-17-cluster10].
+ destroy multi-cluster-service-9794
+ local namespace=multi-cluster-service-9794
+ local ignore_logs=true
+ [[ 1 == 1 ]]
+ echo 'SKIP_DELETE=1, not destroying multi-cluster-service-9794'
SKIP_DELETE=1, not destroying multi-cluster-service-9794
+ return
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------