Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/logs/cross-site-sharded.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod7.0
++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g'
+++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod7.0 'mongod --version'
+++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0
+++ local 'cli=mongod --version'
+++ local pod_name=28607
+++ kubectl_bin -n default run 28607 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gWJDJsI3I0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6ok7THziOR
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default run 28607 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.gWJDJsI3I0
+++ cat /tmp/tmp.6ok7THziOR
+++ rm /tmp/tmp.gWJDJsI3I0 /tmp/tmp.6ok7THziOR
+++ return 0
+++ kubectl_bin -n default wait --for=condition=Ready pod/28607
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WfI3C9USnM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VghWCph3Vn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default wait --for=condition=Ready pod/28607
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.WfI3C9USnM
+++ cat /tmp/tmp.VghWCph3Vn
+++ rm /tmp/tmp.WfI3C9USnM /tmp/tmp.VghWCph3Vn
+++ return 0
++++ kubectl_bin -n default exec 28607 -- bash -c 'mongod --version 2>&1'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.g2aXNNUUgL
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.19VIvjHRjT
++++ local exit_status=0
++++ local timeout=4
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ set +e
++++ kubectl -n default exec 28607 -- bash -c 'mongod --version 2>&1'
++++ exit_status=0
++++ set -e
++++ '[' 0 '!=' 0 -a -n 1 ']'
++++ break
++++ cat /tmp/tmp.g2aXNNUUgL
++++ cat /tmp/tmp.19VIvjHRjT
++++ rm /tmp/tmp.g2aXNNUUgL /tmp/tmp.19VIvjHRjT
++++ return 0
+++ local 'output=db version v7.0.18-11
Build Info: {
    "version": "7.0.18-11",
    "gitVersion": "97ceec31da21050a3f6f3b203bac9e5a2685dcd2",
    "openSSLVersion": "OpenSSL 3.2.2 4 Jun 2024",
    "modules": [],
    "proFeatures": [],
    "allocator": "tcmalloc",
    "environment": {
        "distarch": "x86_64",
        "target_arch": "x86_64"
    }
}'
+++ kubectl_bin -n default delete pod/28607 --grace-period=0 --force
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NyLTKH7m5X
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.UNAIICn7lT
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl -n default delete pod/28607 --grace-period=0 --force
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.NyLTKH7m5X
+++ cat /tmp/tmp.UNAIICn7lT
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+++ rm /tmp/tmp.NyLTKH7m5X /tmp/tmp.UNAIICn7lT
+++ return 0
+++ echo db version v7.0.18-11 Build Info: '{' '"version":' '"7.0.18-11",' '"gitVersion":' '"97ceec31da21050a3f6f3b203bac9e5a2685dcd2",' '"openSSLVersion":' '"OpenSSL' 3.2.2 4 Jun '2024",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}'
++ version_info=7.0.18-11
++ [[ ! 7.0.18-11 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]
++ echo 7.0.18-11
+ FULL_VER=7.0.18-11
+ MONGO_VER=7.0
+ unset OPERATOR_NS
+ main_cluster=cross-site-sharded-main
+ replica_cluster=cross-site-sharded-replica
+ desc 'create main cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create main cluster
-----------------------------------------------------------------------------------

+ create_infra cross-site-sharded-8999
+ local ns=cross-site-sharded-8999
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.2qGkrMzznv
++ mktemp
+ local LAST_ERR=/tmp/tmp.pVvdqaDfwD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2qGkrMzznv
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.pVvdqaDfwD
+ rm /tmp/tmp.2qGkrMzznv /tmp/tmp.pVvdqaDfwD
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.aUTNz2NfCy
++ mktemp
+ local LAST_ERR=/tmp/tmp.NzXRP0n3Ff
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aUTNz2NfCy
+ cat /tmp/tmp.NzXRP0n3Ff
+ rm /tmp/tmp.aUTNz2NfCy /tmp/tmp.NzXRP0n3Ff
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.sNST0IePlX
++ mktemp
+ local LAST_ERR=/tmp/tmp.E0kNYSsgYF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sNST0IePlX
+ cat /tmp/tmp.E0kNYSsgYF
+ rm /tmp/tmp.sNST0IePlX /tmp/tmp.E0kNYSsgYF
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.VeNYbMW4Bq
++ mktemp
+ local LAST_ERR=/tmp/tmp.TgoHlKNzLP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VeNYbMW4Bq
+ cat /tmp/tmp.TgoHlKNzLP
+ rm /tmp/tmp.VeNYbMW4Bq /tmp/tmp.TgoHlKNzLP
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n '' ']'
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.jT5NdorCNf
++ mktemp
+ local LAST_ERR=/tmp/tmp.wAMu814HNa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jT5NdorCNf
role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
serviceaccount "percona-server-mongodb-operator" deleted
rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.wAMu814HNa
+ rm /tmp/tmp.jT5NdorCNf /tmp/tmp.wAMu814HNa
+ return 0
+ check_crd_for_deletion PR-1993-9d545650
+ local git_tag=PR-1993-9d545650
++ yq eval .metadata.name
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1993-9d545650/deploy/crd.yaml
++ /usr/bin/sed s/---//g
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.52UouClmhn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0kHr7SWMee
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.52UouClmhn
++ cat /tmp/tmp.0kHr7SWMee
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.52UouClmhn
++ cat /tmp/tmp.0kHr7SWMee
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.52UouClmhn
++ cat /tmp/tmp.0kHr7SWMee
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.52UouClmhn
++ cat /tmp/tmp.0kHr7SWMee
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.52UouClmhn /tmp/tmp.0kHr7SWMee
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n '' ']'
+ create_namespace cross-site-sharded-8999
+ local namespace=cross-site-sharded-8999
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl api-resources
++ grep chaos-mesh.org
++ awk '{print $1}'
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrolebinding
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces cross-site-sharded-8999'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces cross-site-sharded-8999
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace cross-site-sharded-8999 --ignore-not-found
++ mktemp
+ xargs kubectl delete ns
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
+ local LAST_OUT=/tmp/tmp.U6SQpx6WMI
++ mktemp
+ local LAST_ERR=/tmp/tmp.LFZpVR30wV
+ local exit_status=0
+ local timeout=4
++ mktemp
+ local LAST_OUT=/tmp/tmp.3m1StCHcdS
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace cross-site-sharded-8999 --ignore-not-found
++ mktemp
+ local LAST_ERR=/tmp/tmp.a9TNmcte5O
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.U6SQpx6WMI
+ cat /tmp/tmp.LFZpVR30wV
+ rm /tmp/tmp.U6SQpx6WMI /tmp/tmp.LFZpVR30wV
+ return 0
+ kubectl_bin wait --for=delete namespace cross-site-sharded-8999
++ mktemp
+ local LAST_OUT=/tmp/tmp.YGXTZp7E0M
++ mktemp
+ local LAST_ERR=/tmp/tmp.rRuTwVn3XY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace cross-site-sharded-8999
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3m1StCHcdS
+ cat /tmp/tmp.a9TNmcte5O
+ rm /tmp/tmp.3m1StCHcdS /tmp/tmp.a9TNmcte5O
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YGXTZp7E0M
+ cat /tmp/tmp.rRuTwVn3XY
+ rm /tmp/tmp.YGXTZp7E0M /tmp/tmp.rRuTwVn3XY
+ return 0
+ desc 'create namespace cross-site-sharded-8999'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace cross-site-sharded-8999
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cross-site-sharded-8999
++ mktemp
+ local LAST_OUT=/tmp/tmp.DnpHTJzU65
++ mktemp
+ local LAST_ERR=/tmp/tmp.VRwlm8QTLS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cross-site-sharded-8999
namespace "cross-site-sharded-25359" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DnpHTJzU65
namespace "cross-site-sharded-replica-32094" deleted
namespace/cross-site-sharded-8999 created
+ cat /tmp/tmp.VRwlm8QTLS
+ rm /tmp/tmp.DnpHTJzU65 /tmp/tmp.VRwlm8QTLS
+ return 0
+ set_kube_ctx cross-site-sharded-8999
+ local namespace=cross-site-sharded-8999
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h0FMcGXwiu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eekdErXHPC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.h0FMcGXwiu
++ cat /tmp/tmp.eekdErXHPC
++ rm /tmp/tmp.h0FMcGXwiu /tmp/tmp.eekdErXHPC
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1993-9d545650-2-cluster3 --namespace=cross-site-sharded-8999
++ mktemp
+ local LAST_OUT=/tmp/tmp.iea72G7nkS
++ mktemp
+ local LAST_ERR=/tmp/tmp.lpCWtn4NLJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1993-9d545650-2-cluster3 --namespace=cross-site-sharded-8999
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iea72G7nkS
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1993-9d545650-2-cluster3" modified.
+ cat /tmp/tmp.lpCWtn4NLJ
+ rm /tmp/tmp.iea72G7nkS /tmp/tmp.lpCWtn4NLJ
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.84frEyTRtD
++ mktemp
+ local LAST_ERR=/tmp/tmp.XzmIqSRZEG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.84frEyTRtD
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.XzmIqSRZEG
+ rm /tmp/tmp.84frEyTRtD /tmp/tmp.XzmIqSRZEG
+ return 0
+ '[' -n '' ']'
+ apply_rbac rbac
+ local operator_namespace=psmdb-operator
+ local rbac=rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.R7R4EUjZzN
++ mktemp
+ local LAST_ERR=/tmp/tmp.X6zDv3MSYt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.R7R4EUjZzN
role.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.X6zDv3MSYt
+ rm /tmp/tmp.R7R4EUjZzN /tmp/tmp.X6zDv3MSYt
+ return 0
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1993-9d545650") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/deploy/operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CsWu8BI80d
++ mktemp
+ local LAST_ERR=/tmp/tmp.ojUQZCPqQv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CsWu8BI80d
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.ojUQZCPqQv
+ rm /tmp/tmp.CsWu8BI80d /tmp/tmp.ojUQZCPqQv
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s5HPRRQt4t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1bgF4rIwCI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s5HPRRQt4t
++ cat /tmp/tmp.1bgF4rIwCI
++ rm /tmp/tmp.s5HPRRQt4t /tmp/tmp.1bgF4rIwCI
++ return 0
+ wait_pod percona-server-mongodb-operator-5bb9fd6ff9-b9c44
+ local pod=percona-server-mongodb-operator-5bb9fd6ff9-b9c44
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5bb9fd6ff9-b9c44 to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eRDJCC6F1f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TrhOu5Om8g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eRDJCC6F1f
++ cat /tmp/tmp.TrhOu5Om8g
++ rm /tmp/tmp.eRDJCC6F1f /tmp/tmp.TrhOu5Om8g
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-5bb9fd6ff9-b9c44
++ mktemp
+ local LAST_OUT=/tmp/tmp.AECNkuWsYY
++ mktemp
+ local LAST_ERR=/tmp/tmp.D9wpm3wMRl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-5bb9fd6ff9-b9c44
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AECNkuWsYY
+ cat /tmp/tmp.D9wpm3wMRl
+ rm /tmp/tmp.AECNkuWsYY /tmp/tmp.D9wpm3wMRl
+ return 0
2025-07-04T00:55:47.198Z	INFO	setup	Manager starting up	{"gitCommit": "9d5456508def1bb97ffa7da79b450ff41a98b5fb", "gitBranch": "PR-1993-9d545650", "buildTime": "", "goVersion": "go1.24.4", "os": "linux", "arch": "amd64"}
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.W9JqiSXGGR
++ mktemp
+ local LAST_ERR=/tmp/tmp.MurVYG4OQk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.W9JqiSXGGR
deployment.apps/psmdb-client created
secret/cross-site-sharded-main-secrets created
secret/cross-site-sharded-main-ssl created
secret/cross-site-sharded-main-ssl-internal created
+ cat /tmp/tmp.MurVYG4OQk
+ rm /tmp/tmp.W9JqiSXGGR /tmp/tmp.MurVYG4OQk
+ return 0
+ desc 'create main PSMDB cluster cross-site-sharded-main.'
+ set +o xtrace


-----------------------------------------------------------------------------------
create main PSMDB cluster cross-site-sharded-main.
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1993-9d545650"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.botTBnVlxy
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.EDg4cnQvXw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.botTBnVlxy
perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created
+ cat /tmp/tmp.EDg4cnQvXw
+ rm /tmp/tmp.botTBnVlxy /tmp/tmp.EDg4cnQvXw
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running cross-site-sharded-main-rs0 3
+ local name=cross-site-sharded-main-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=cross-site-sharded-main
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-main-rs0-0
+ local pod=cross-site-sharded-main-rs0-0
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-0 to be ready.........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-main-rs0-1
+ local pod=cross-site-sharded-main-rs0-1
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-1 to be ready...........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.abwFUh9ed6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KXq0U0qT5T
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.abwFUh9ed6
++ cat /tmp/tmp.KXq0U0qT5T
++ rm /tmp/tmp.abwFUh9ed6 /tmp/tmp.KXq0U0qT5T
++ return 0
+ [[ false == \t\r\u\e ]]
+ wait_pod cross-site-sharded-main-rs0-2
+ local pod=cross-site-sharded-main-rs0-2
+ set +o xtrace
waiting for pod/cross-site-sharded-main-rs0-2 to be ready............OK
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EPgezzMMu3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AaMZy9zZMR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EPgezzMMu3
++ cat /tmp/tmp.AaMZy9zZMR
++ rm /tmp/tmp.EPgezzMMu3 /tmp/tmp.AaMZy9zZMR
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wK1VyWiekc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uuZFglDs36
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wK1VyWiekc
++ cat /tmp/tmp.uuZFglDs36
++ rm /tmp/tmp.wK1VyWiekc /tmp/tmp.uuZFglDs36
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness............
+ wait_for_running cross-site-sharded-main-cfg 3 false
+ local name=cross-site-sharded-main-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=cross-site-sharded-main
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod cross-site-sharded-main-cfg-0
+ local pod=cross-site-sharded-main-cfg-0
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod cross-site-sharded-main-cfg-1
+ local pod=cross-site-sharded-main-cfg-1
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ewqYrwrLcx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H6rFLHJHaN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ewqYrwrLcx
++ cat /tmp/tmp.H6rFLHJHaN
++ rm /tmp/tmp.ewqYrwrLcx /tmp/tmp.H6rFLHJHaN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod cross-site-sharded-main-cfg-2
+ local pod=cross-site-sharded-main-cfg-2
+ set +o xtrace
waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fkhGvzrsZk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jP2Tk6NZMK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fkhGvzrsZk
++ cat /tmp/tmp.jP2Tk6NZMK
++ rm /tmp/tmp.fkhGvzrsZk /tmp/tmp.jP2Tk6NZMK
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UNeLXMYD0b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0P4V3NLNKC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UNeLXMYD0b
++ cat /tmp/tmp.0P4V3NLNKC
++ rm /tmp/tmp.UNeLXMYD0b /tmp/tmp.0P4V3NLNKC
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ desc 'create user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uV1HnAu6jX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K1xp4YsW3K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uV1HnAu6jX
++ cat /tmp/tmp.K1xp4YsW3K
++ rm /tmp/tmp.uV1HnAu6jX /tmp/tmp.K1xp4YsW3K
++ return 0
+ local client_container=psmdb-client-66f577db5f-lxnk8
+ kubectl_bin exec psmdb-client-66f577db5f-lxnk8 -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6fPAemePMX
++ mktemp
+ local LAST_ERR=/tmp/tmp.4rHxaJxTar
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-lxnk8 -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6fPAemePMX
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("693232d4-6ec2-468a-90aa-14e12d64e834") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "user",
	"roles" : [
		{
			"db" : "app",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.4rHxaJxTar
+ rm /tmp/tmp.6fPAemePMX /tmp/tmp.4rHxaJxTar
+ return 0
+ sleep 2
+ desc 'set chunk size to 2 MB'
+ set +o xtrace


-----------------------------------------------------------------------------------
set chunk size to 2 MB
-----------------------------------------------------------------------------------

+ run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )'
+ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M31HQ8YXUh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A8RVDK6Dvg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.M31HQ8YXUh
++ cat /tmp/tmp.A8RVDK6Dvg
++ rm /tmp/tmp.M31HQ8YXUh /tmp/tmp.A8RVDK6Dvg
++ return 0
+ local client_container=psmdb-client-66f577db5f-lxnk8
+ kubectl_bin exec psmdb-client-66f577db5f-lxnk8 -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.xoG7NDNfk6
++ mktemp
+ local LAST_ERR=/tmp/tmp.zx6Kznipp3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-66f577db5f-lxnk8 -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xoG7NDNfk6
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-8999.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("89217fb2-d2d4-4145-aa58-af50120561c4") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db config
WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" })
bye
+ cat /tmp/tmp.zx6Kznipp3
+ rm /tmp/tmp.xoG7NDNfk6 /tmp/tmp.zx6Kznipp3
+ return 0
+ sleep 2
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js
+ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-8999
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local mongo_bin=mongo
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nG3dOr4CXy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8PQkJgmbZY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nG3dOr4CXy
++ cat /tmp/tmp.8PQkJgmbZY
++ rm /tmp/tmp.nG3dOr4CXy /tmp/tmp.8PQkJgmbZY
++ return 0
+ local client_container=psmdb-client-66f577db5f-lxnk8
++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js
+ name=data.js
+ kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js cross-site-sharded-8999/psmdb-client-66f577db5f-lxnk8:/tmp
++ mktemp
+ local LAST_OUT=/tmp/tmp.s9RzOhZRnU
++ mktemp
+ local LAST_ERR=/tmp/tmp.9qQeWv8HOA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js cross-site-sharded-8999/psmdb-client-66f577db5f-lxnk8:/tmp
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.s9RzOhZRnU
+ cat /tmp/tmp.9qQeWv8HOA
error: context deadline exceeded
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js cross-site-sharded-8999/psmdb-client-66f577db5f-lxnk8:/tmp
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.s9RzOhZRnU
+ cat /tmp/tmp.9qQeWv8HOA
error: context deadline exceeded
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1993/e2e-tests/cross-site-sharded/data.js cross-site-sharded-8999/psmdb-client-66f577db5f-lxnk8:/tmp
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.s9RzOhZRnU
+ cat /tmp/tmp.9qQeWv8HOA
error: context deadline exceeded
+ sleep 8
+ cat /tmp/tmp.s9RzOhZRnU
+ cat /tmp/tmp.9qQeWv8HOA
error: context deadline exceeded
+ rm /tmp/tmp.s9RzOhZRnU /tmp/tmp.9qQeWv8HOA
+ return 1