Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/logs/cross-site-sharded.log Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.31) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod8.0 ++ local image=perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod8.0 'mongod --version' +++ local image=perconalab/percona-server-mongodb-operator:main-mongod8.0 +++ local 'cli=mongod --version' +++ local pod_name=7850 +++ /usr/sbin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' +++ kubectl_bin -n default run 7850 --image=perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FX193HNy4B ++++ mktemp +++ local LAST_ERR=/tmp/tmp.43heD0XwUQ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default run 7850 --image=perconalab/percona-server-mongodb-operator:main-mongod8.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FX193HNy4B +++ cat /tmp/tmp.43heD0XwUQ +++ rm /tmp/tmp.FX193HNy4B /tmp/tmp.43heD0XwUQ +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/7850 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1MZ8ebbqUM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wCrZgGOxOe +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/7850 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1MZ8ebbqUM +++ cat /tmp/tmp.wCrZgGOxOe +++ rm /tmp/tmp.1MZ8ebbqUM /tmp/tmp.wCrZgGOxOe +++ return 0 ++++ kubectl_bin -n default exec 7850 -- bash -c 'mongod --version 2>&1' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9jZKBGJEFJ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.3jL3HRYjoE ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl -n default exec 7850 -- bash -c 'mongod --version 2>&1' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.9jZKBGJEFJ ++++ cat /tmp/tmp.3jL3HRYjoE ++++ rm /tmp/tmp.9jZKBGJEFJ /tmp/tmp.3jL3HRYjoE ++++ return 0 +++ local 'output=db version v8.0.17-6 Build Info: { "version": "8.0.17-6", "gitVersion": "22257a4d2e901468cf31d122e1c0ea549b66770e", "openSSLVersion": "OpenSSL 3.5.1 1 Jul 2025", "modules": [], "perconaFeatures": [ "MemoryEngine", "HotBackup", "BackupCursorAggregationStage", "BackupCursorExtendAggregationStage", "AWSIAM", "Kerberos", "LDAP", "OIDC", "TDE", "FIPSMode", "FCBIS", "Auditing", "ProfilingRateLimit", "LogRedaction", "ngram" ], "allocator": "tcmalloc-google", "environment": { "distarch": "x86_64", "target_arch": "x86_64" } }' +++ kubectl_bin -n default delete pod/7850 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JRM5yWDnNN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kSPJLE3CgX +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl -n default delete pod/7850 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.JRM5yWDnNN +++ cat /tmp/tmp.kSPJLE3CgX Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.JRM5yWDnNN /tmp/tmp.kSPJLE3CgX +++ return 0 +++ echo db version v8.0.17-6 Build Info: '{' '"version":' '"8.0.17-6",' '"gitVersion":' '"22257a4d2e901468cf31d122e1c0ea549b66770e",' '"openSSLVersion":' '"OpenSSL' 3.5.1 1 Jul '2025",' '"modules":' '[],' '"perconaFeatures":' '[' '"MemoryEngine",' '"HotBackup",' '"BackupCursorAggregationStage",' '"BackupCursorExtendAggregationStage",' '"AWSIAM",' '"Kerberos",' '"LDAP",' '"OIDC",' '"TDE",' '"FIPSMode",' '"FCBIS",' '"Auditing",' '"ProfilingRateLimit",' '"LogRedaction",' '"ngram"' '],' '"allocator":' '"tcmalloc-google",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=8.0.17-6 ++ [[ ! 8.0.17-6 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 8.0.17-6 + FULL_VER=8.0.17-6 + MONGO_VER=8.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-14047 + local ns=cross-site-sharded-14047 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.2kqJMMiSrY ++ mktemp + local LAST_ERR=/tmp/tmp.3Wi4TTMmNa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2kqJMMiSrY customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.3Wi4TTMmNa + rm /tmp/tmp.2kqJMMiSrY /tmp/tmp.3Wi4TTMmNa + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.rJXqDk0NvX ++ mktemp + local LAST_ERR=/tmp/tmp.1CPoLrv4nA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rJXqDk0NvX + cat /tmp/tmp.1CPoLrv4nA + rm /tmp/tmp.rJXqDk0NvX /tmp/tmp.1CPoLrv4nA + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.DNYOueZS3l ++ mktemp + local LAST_ERR=/tmp/tmp.5GZ79cUAoB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DNYOueZS3l + cat /tmp/tmp.5GZ79cUAoB + rm /tmp/tmp.DNYOueZS3l /tmp/tmp.5GZ79cUAoB + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.EcIkoGX9qb ++ mktemp + local LAST_ERR=/tmp/tmp.hnKrU1O6E4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EcIkoGX9qb + cat /tmp/tmp.hnKrU1O6E4 + rm /tmp/tmp.EcIkoGX9qb /tmp/tmp.hnKrU1O6E4 + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.AWgbmKd1jR ++ mktemp + local LAST_ERR=/tmp/tmp.0SynYny3We + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AWgbmKd1jR serviceaccount "percona-server-mongodb-operator" deleted from cross-site-sharded-7949 namespace rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted from cross-site-sharded-7949 namespace + cat /tmp/tmp.0SynYny3We + rm /tmp/tmp.AWgbmKd1jR /tmp/tmp.0SynYny3We + return 0 + check_crd_for_deletion PR-1961-7f9b8fd6 + local git_tag=PR-1961-7f9b8fd6 ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1961-7f9b8fd6/deploy/crd.yaml ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FAG5pNvGDp +++ mktemp ++ local LAST_ERR=/tmp/tmp.KYfuhTqX5h ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.FAG5pNvGDp ++ cat /tmp/tmp.KYfuhTqX5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.FAG5pNvGDp ++ cat /tmp/tmp.KYfuhTqX5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.FAG5pNvGDp ++ cat /tmp/tmp.KYfuhTqX5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.FAG5pNvGDp ++ cat /tmp/tmp.KYfuhTqX5h Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.FAG5pNvGDp /tmp/tmp.KYfuhTqX5h ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n '' ']' + create_namespace cross-site-sharded-14047 + local namespace=cross-site-sharded-14047 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces cross-site-sharded-14047' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-14047 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-14047 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.1AG0DJtHEN ++ mktemp + local LAST_OUT=/tmp/tmp.wt2WsY0xj1 + local LAST_ERR=/tmp/tmp.28PCgYuJKX ++ mktemp + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.LlFWmW4CnN + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace cross-site-sharded-14047 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1AG0DJtHEN + cat /tmp/tmp.28PCgYuJKX + rm /tmp/tmp.1AG0DJtHEN /tmp/tmp.28PCgYuJKX + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wt2WsY0xj1 + cat /tmp/tmp.LlFWmW4CnN + rm /tmp/tmp.wt2WsY0xj1 /tmp/tmp.LlFWmW4CnN + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-14047 ++ mktemp + local LAST_OUT=/tmp/tmp.NGg0iw9WX3 ++ mktemp + local LAST_ERR=/tmp/tmp.Mz3aAzX5zz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace cross-site-sharded-14047 namespace "cross-site-sharded-7949" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NGg0iw9WX3 + cat /tmp/tmp.Mz3aAzX5zz + rm /tmp/tmp.NGg0iw9WX3 /tmp/tmp.Mz3aAzX5zz + return 0 + desc 'create namespace cross-site-sharded-14047' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-14047 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-14047 ++ mktemp + local LAST_OUT=/tmp/tmp.T1DxZyP3nB ++ mktemp + local LAST_ERR=/tmp/tmp.8Lw2PuAtut + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace cross-site-sharded-14047 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T1DxZyP3nB namespace/cross-site-sharded-14047 created + cat /tmp/tmp.8Lw2PuAtut + rm /tmp/tmp.T1DxZyP3nB /tmp/tmp.8Lw2PuAtut + return 0 + set_kube_ctx cross-site-sharded-14047 + local namespace=cross-site-sharded-14047 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.J3ATxnmeqt +++ mktemp ++ local LAST_ERR=/tmp/tmp.KXeb1QhFhC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J3ATxnmeqt ++ cat /tmp/tmp.KXeb1QhFhC ++ rm /tmp/tmp.J3ATxnmeqt /tmp/tmp.KXeb1QhFhC ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster3 --namespace=cross-site-sharded-14047 ++ mktemp + local LAST_OUT=/tmp/tmp.J4zhBZXjEg ++ mktemp + local LAST_ERR=/tmp/tmp.Lc5eztbR16 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster3 --namespace=cross-site-sharded-14047 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.J4zhBZXjEg Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1961-7f9b8fd6-3-cluster3" modified. + cat /tmp/tmp.Lc5eztbR16 + rm /tmp/tmp.J4zhBZXjEg /tmp/tmp.Lc5eztbR16 + return 0 + deploy_operator + desc 'start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.N3ymsJJQ4j ++ mktemp + local LAST_ERR=/tmp/tmp.R44iFE63QN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.N3ymsJJQ4j customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.R44iFE63QN + rm /tmp/tmp.N3ymsJJQ4j /tmp/tmp.R44iFE63QN + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.LUEVpQUiJm ++ mktemp + local LAST_ERR=/tmp/tmp.9mJh1bsEgm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LUEVpQUiJm role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.9mJh1bsEgm + rm /tmp/tmp.LUEVpQUiJm /tmp/tmp.9mJh1bsEgm + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/deploy/operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.5IKfgENOSi ++ mktemp + local LAST_ERR=/tmp/tmp.ReeDYzifoA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5IKfgENOSi deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.ReeDYzifoA + rm /tmp/tmp.5IKfgENOSi /tmp/tmp.ReeDYzifoA + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lBAxNkJ5mi +++ mktemp ++ local LAST_ERR=/tmp/tmp.EvXhfhHtC3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lBAxNkJ5mi ++ cat /tmp/tmp.EvXhfhHtC3 ++ rm /tmp/tmp.lBAxNkJ5mi /tmp/tmp.EvXhfhHtC3 ++ return 0 + wait_operator_pod percona-server-mongodb-operator-6db5675b97-br5v2 + local pod=percona-server-mongodb-operator-6db5675b97-br5v2 + set +o xtrace waiting for pod/percona-server-mongodb-operator-6db5675b97-br5v2 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OrxgpLmD32 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xO8HgFHZfx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OrxgpLmD32 ++ cat /tmp/tmp.xO8HgFHZfx ++ rm /tmp/tmp.OrxgpLmD32 /tmp/tmp.xO8HgFHZfx ++ return 0 + kubectl_bin logs percona-server-mongodb-operator-6db5675b97-br5v2 ++ mktemp + local LAST_OUT=/tmp/tmp.P1wCkXhkm6 ++ mktemp + local LAST_ERR=/tmp/tmp.3dEa2uMSAB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs percona-server-mongodb-operator-6db5675b97-br5v2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.P1wCkXhkm6 + cat /tmp/tmp.3dEa2uMSAB + rm /tmp/tmp.P1wCkXhkm6 /tmp/tmp.3dEa2uMSAB + return 0 2026-01-12T21:09:27.079Z INFO setup Manager starting up {"gitCommit": "7f9b8fd660ef537a8ff13f3505cffba918719e96", "gitBranch": "PR-1961-7f9b8fd6", "buildTime": "", "goVersion": "go1.25.5", "os": "linux", "arch": "amd64"} + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.zNzjL086S6 ++ mktemp + local LAST_ERR=/tmp/tmp.8JcVsCZMX9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zNzjL086S6 deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.8JcVsCZMX9 + rm /tmp/tmp.zNzjL086S6 /tmp/tmp.8JcVsCZMX9 + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1961-7f9b8fd6"' + /usr/sbin/sed -e s/NAME_SPACE/cross-site-sharded-14047/g + local LAST_OUT=/tmp/tmp.fk3lgyBhAA + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.KoYckdUSfh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fk3lgyBhAA perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.KoYckdUSfh + rm /tmp/tmp.fk3lgyBhAA /tmp/tmp.KoYckdUSfh + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready......................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready...........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.avJMHdAPKv +++ mktemp ++ local LAST_ERR=/tmp/tmp.LEenWWgBaE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.avJMHdAPKv ++ cat /tmp/tmp.LEenWWgBaE ++ rm /tmp/tmp.avJMHdAPKv /tmp/tmp.LEenWWgBaE ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready..............OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PpNqghFCsJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.YgNpKv4hYF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PpNqghFCsJ ++ cat /tmp/tmp.YgNpKv4hYF ++ rm /tmp/tmp.PpNqghFCsJ /tmp/tmp.YgNpKv4hYF ++ return 0 + [[ false == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uFVw7VcWKm +++ mktemp ++ local LAST_ERR=/tmp/tmp.KsBElNSljs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uFVw7VcWKm ++ cat /tmp/tmp.KsBElNSljs ++ rm /tmp/tmp.uFVw7VcWKm /tmp/tmp.KsBElNSljs ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.................................................... + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dmmfJX8BaF +++ mktemp ++ local LAST_ERR=/tmp/tmp.6dtdSuYdG3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dmmfJX8BaF ++ cat /tmp/tmp.6dtdSuYdG3 ++ rm /tmp/tmp.dmmfJX8BaF /tmp/tmp.6dtdSuYdG3 ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lsljWqPcLh +++ mktemp ++ local LAST_ERR=/tmp/tmp.YHX9ijU2OD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lsljWqPcLh ++ cat /tmp/tmp.YHX9ijU2OD ++ rm /tmp/tmp.lsljWqPcLh /tmp/tmp.YHX9ijU2OD ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MSyuUDpA8P +++ mktemp ++ local LAST_ERR=/tmp/tmp.M6aGCvi14W ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MSyuUDpA8P ++ cat /tmp/tmp.M6aGCvi14W ++ rm /tmp/tmp.MSyuUDpA8P /tmp/tmp.M6aGCvi14W ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9bFd1xCZYj +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nyhgvjdrys ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9bFd1xCZYj ++ cat /tmp/tmp.Nyhgvjdrys ++ rm /tmp/tmp.9bFd1xCZYj /tmp/tmp.Nyhgvjdrys ++ return 0 + local client_container=psmdb-client-696897d69b-4jhtn + kubectl_bin exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.vNR9R4hpRK ++ mktemp + local LAST_ERR=/tmp/tmp.ORm6WBZcLs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vNR9R4hpRK Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("0cf89438-1aa3-4817-a26c-4815d3cdf866") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.ORm6WBZcLs + rm /tmp/tmp.vNR9R4hpRK /tmp/tmp.ORm6WBZcLs + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1DHIPyTSv0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.51mtYGo0kH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1DHIPyTSv0 ++ cat /tmp/tmp.51mtYGo0kH ++ rm /tmp/tmp.1DHIPyTSv0 /tmp/tmp.51mtYGo0kH ++ return 0 + local client_container=psmdb-client-696897d69b-4jhtn + kubectl_bin exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.AznPoCBwIo ++ mktemp + local LAST_ERR=/tmp/tmp.ogOFEmdWPo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AznPoCBwIo Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1ec15add-746c-478e-a378-9d7e75d2c6e5") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.ogOFEmdWPo + rm /tmp/tmp.AznPoCBwIo /tmp/tmp.ogOFEmdWPo + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local mongo_bin=mongo ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aeFyFAD7eL +++ mktemp ++ local LAST_ERR=/tmp/tmp.pS5QmpxYNK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aeFyFAD7eL ++ cat /tmp/tmp.pS5QmpxYNK ++ rm /tmp/tmp.aeFyFAD7eL /tmp/tmp.pS5QmpxYNK ++ return 0 + local client_container=psmdb-client-696897d69b-4jhtn ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/data.js cross-site-sharded-14047/psmdb-client-696897d69b-4jhtn:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.nSutHySDIR ++ mktemp + local LAST_ERR=/tmp/tmp.QZPKMsoQmV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1961/e2e-tests/cross-site-sharded/data.js cross-site-sharded-14047/psmdb-client-696897d69b-4jhtn:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nSutHySDIR + cat /tmp/tmp.QZPKMsoQmV + rm /tmp/tmp.nSutHySDIR /tmp/tmp.QZPKMsoQmV + return 0 + kubectl_bin exec psmdb-client-696897d69b-4jhtn -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.2t2HxAHHpJ ++ mktemp + local LAST_ERR=/tmp/tmp.KwyustPHGE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-696897d69b-4jhtn -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2t2HxAHHpJ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("638733b3-d7bc-4788-91db-d82fc7cfaaf0") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match + cat /tmp/tmp.KwyustPHGE + rm /tmp/tmp.2t2HxAHHpJ /tmp/tmp.KwyustPHGE + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xs4KXdmWGf +++ mktemp ++ local LAST_ERR=/tmp/tmp.ag3TlzKXg5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xs4KXdmWGf ++ cat /tmp/tmp.ag3TlzKXg5 ++ rm /tmp/tmp.xs4KXdmWGf /tmp/tmp.ag3TlzKXg5 ++ return 0 + local client_container=psmdb-client-696897d69b-4jhtn + kubectl_bin exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.xNqeQuNXWJ ++ mktemp + local LAST_ERR=/tmp/tmp.Jq3I2X9Qlq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xNqeQuNXWJ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ddcab570-6104-481a-9952-08c254e9cf98") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1768252473, 1), "signature" : { "hash" : BinData(0,"Fa06YuJB1ygWL/K8TduHhMxtnmA="), "keyId" : NumberLong("7594585709382467589") } }, "operationTime" : Timestamp(1768252473, 1) } bye + cat /tmp/tmp.Jq3I2X9Qlq + rm /tmp/tmp.xNqeQuNXWJ /tmp/tmp.Jq3I2X9Qlq + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27017 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DjszmwlQcy +++ mktemp ++ local LAST_ERR=/tmp/tmp.MQ2PjNIxBS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.DjszmwlQcy ++ cat /tmp/tmp.MQ2PjNIxBS ++ rm /tmp/tmp.DjszmwlQcy /tmp/tmp.MQ2PjNIxBS ++ return 0 + local client_container=psmdb-client-696897d69b-4jhtn + kubectl_bin exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.ZMqkV05bEM ++ mktemp + local LAST_ERR=/tmp/tmp.W6oGQGiLpx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-696897d69b-4jhtn -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZMqkV05bEM Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bd98c9c3-f051-445d-92a3-c219c3c72526") } Percona Server for MongoDB server version: v8.0.17-6 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1768252480, 21), "signature" : { "hash" : BinData(0,"lDewiqd4PmrtdE6RdFEo83iQ02w="), "keyId" : NumberLong("7594585709382467589") } }, "operationTime" : Timestamp(1768252480, 20) } bye + cat /tmp/tmp.W6oGQGiLpx + rm /tmp/tmp.ZMqkV05bEM /tmp/tmp.W6oGQGiLpx + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 8.0 != \4\.\4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047 ++ grep 'switched to db app' -A 1 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local port=27017 ++ local mongo_bin=mongo ++ grep -v 'switched to db app' +++ echo .svc.cluster.local +++ awk -F: '{print $2}' ++ suffix_port= ++ [[ -z '' ]] ++ suffix=.svc.cluster.local:27017 +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SiqhwHbuyk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GKaij9n63J +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.SiqhwHbuyk +++ cat /tmp/tmp.GKaij9n63J Unable to connect to the server: net/http: TLS handshake timeout error executing jsonpath "{.items[].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}} +++ sleep 0 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.SiqhwHbuyk +++ cat /tmp/tmp.GKaij9n63J Unable to connect to the server: dial tcp 34.58.199.54:443: i/o timeout error executing jsonpath "{.items[].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}} +++ sleep 4 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=1 +++ set -e +++ '[' 1 '!=' 0 -a -n 1 ']' +++ cat /tmp/tmp.SiqhwHbuyk +++ cat /tmp/tmp.GKaij9n63J Unable to connect to the server: net/http: TLS handshake timeout error executing jsonpath "{.items[].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}} +++ sleep 8 +++ cat /tmp/tmp.SiqhwHbuyk +++ cat /tmp/tmp.GKaij9n63J Unable to connect to the server: net/http: TLS handshake timeout error executing jsonpath "{.items[].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}} +++ rm /tmp/tmp.SiqhwHbuyk /tmp/tmp.GKaij9n63J +++ return 1 ++ local client_container= ++ kubectl_bin exec -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jFgxv5uG8i +++ mktemp ++ local LAST_ERR=/tmp/tmp.FwVK5zn1XX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.jFgxv5uG8i ++ cat /tmp/tmp.FwVK5zn1XX error: pod, type/name or --filename must be specified ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.jFgxv5uG8i ++ cat /tmp/tmp.FwVK5zn1XX error: pod, type/name or --filename must be specified ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-14047.svc.cluster.local:27017/admin ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.jFgxv5uG8i ++ cat /tmp/tmp.FwVK5zn1XX error: pod, type/name or --filename must be specified ++ sleep 8 ++ cat /tmp/tmp.jFgxv5uG8i ++ cat /tmp/tmp.FwVK5zn1XX error: pod, type/name or --filename must be specified ++ rm /tmp/tmp.jFgxv5uG8i /tmp/tmp.FwVK5zn1XX ++ return 1 + chunks_param2=