Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/logs/cross-site-sharded.log WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1 ++ get_mongod_ver_from_image perconalab/percona-server-mongodb-operator:main-mongod7.0 ++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0 +++ /usr/bin/sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g' +++ run_simple_cli_inside_image perconalab/percona-server-mongodb-operator:main-mongod7.0 'mongod --version' +++ local image=perconalab/percona-server-mongodb-operator:main-mongod7.0 +++ local 'cli=mongod --version' +++ local pod_name=26544 +++ kubectl_bin -n default run 26544 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity ++++ mktemp +++ local LAST_OUT=/tmp/tmp.KsJXp5tOwq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FIz8Dd480r +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default run 26544 --image=perconalab/percona-server-mongodb-operator:main-mongod7.0 --restart=Never --command -- sleep infinity +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.KsJXp5tOwq +++ cat /tmp/tmp.FIz8Dd480r +++ rm /tmp/tmp.KsJXp5tOwq /tmp/tmp.FIz8Dd480r +++ return 0 +++ kubectl_bin -n default wait --for=condition=Ready pod/26544 ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FsaDW6T7PW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XMBtTEs3Jg +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default wait --for=condition=Ready pod/26544 +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FsaDW6T7PW +++ cat /tmp/tmp.XMBtTEs3Jg +++ rm /tmp/tmp.FsaDW6T7PW /tmp/tmp.XMBtTEs3Jg +++ return 0 ++++ kubectl_bin -n default exec 26544 -- mongod --version +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.AK7H8fbSAc +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.KsDPZDxgU7 ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl -n default exec 26544 -- mongod --version ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.AK7H8fbSAc ++++ cat /tmp/tmp.KsDPZDxgU7 ++++ rm /tmp/tmp.AK7H8fbSAc /tmp/tmp.KsDPZDxgU7 ++++ return 0 +++ local 'output=db version v7.0.14-8 Build Info: { "version": "7.0.14-8", "gitVersion": "4479fd75bea6bf98a934c29007352dd36f3e1db5", "openSSLVersion": "OpenSSL 1.1.1k FIPS 25 Mar 2021", "modules": [], "proFeatures": [], "allocator": "tcmalloc", "environment": { "distarch": "x86_64", "target_arch": "x86_64" } }' +++ kubectl_bin -n default delete pod/26544 --grace-period=0 --force ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GCau26VaPC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uNnusnryoC +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl -n default delete pod/26544 --grace-period=0 --force +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.GCau26VaPC +++ cat /tmp/tmp.uNnusnryoC Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. +++ rm /tmp/tmp.GCau26VaPC /tmp/tmp.uNnusnryoC +++ return 0 +++ echo db version v7.0.14-8 Build Info: '{' '"version":' '"7.0.14-8",' '"gitVersion":' '"4479fd75bea6bf98a934c29007352dd36f3e1db5",' '"openSSLVersion":' '"OpenSSL' 1.1.1k FIPS 25 Mar '2021",' '"modules":' '[],' '"proFeatures":' '[],' '"allocator":' '"tcmalloc",' '"environment":' '{' '"distarch":' '"x86_64",' '"target_arch":' '"x86_64"' '}' '}' ++ version_info=7.0.14-8 ++ [[ ! 7.0.14-8 =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]] ++ echo 7.0.14-8 + FULL_VER=7.0.14-8 + MONGO_VER=7.0 + unset OPERATOR_NS + main_cluster=cross-site-sharded-main + replica_cluster=cross-site-sharded-replica + desc 'create main cluster' + set +o xtrace ----------------------------------------------------------------------------------- create main cluster ----------------------------------------------------------------------------------- + create_infra cross-site-sharded-2508 + local ns=cross-site-sharded-2508 + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.X2TgeakKBu ++ mktemp + local LAST_ERR=/tmp/tmp.NMYKpqu9A3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.X2TgeakKBu customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.NMYKpqu9A3 + rm /tmp/tmp.X2TgeakKBu /tmp/tmp.NMYKpqu9A3 + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.BVIqlrw89r ++ mktemp + local LAST_ERR=/tmp/tmp.cBgdbCVRso + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.BVIqlrw89r + cat /tmp/tmp.cBgdbCVRso + rm /tmp/tmp.BVIqlrw89r /tmp/tmp.cBgdbCVRso + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.d8HnhkwTlo ++ mktemp + local LAST_ERR=/tmp/tmp.yVDoE1xJmT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.d8HnhkwTlo + cat /tmp/tmp.yVDoE1xJmT + rm /tmp/tmp.d8HnhkwTlo /tmp/tmp.yVDoE1xJmT + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.w6TtKCI2sv ++ mktemp + local LAST_ERR=/tmp/tmp.7tiiqkVjtZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.w6TtKCI2sv + cat /tmp/tmp.7tiiqkVjtZ + rm /tmp/tmp.w6TtKCI2sv /tmp/tmp.7tiiqkVjtZ + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.GA95CveoTz ++ mktemp + local LAST_ERR=/tmp/tmp.YBOfn5AO0W + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GA95CveoTz role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted serviceaccount "percona-server-mongodb-operator" deleted rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.YBOfn5AO0W + rm /tmp/tmp.GA95CveoTz /tmp/tmp.YBOfn5AO0W + return 0 + check_crd_for_deletion PR-1647-9959fb75 + local git_tag=PR-1647-9959fb75 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1647-9959fb75/deploy/crd.yaml ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ yq eval .metadata.name ++ /usr/bin/sed s/---//g + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SCz3hrheEF +++ mktemp ++ local LAST_ERR=/tmp/tmp.X7FQklnPon ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.SCz3hrheEF ++ cat /tmp/tmp.X7FQklnPon Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.SCz3hrheEF ++ cat /tmp/tmp.X7FQklnPon Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.SCz3hrheEF ++ cat /tmp/tmp.X7FQklnPon Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.SCz3hrheEF ++ cat /tmp/tmp.X7FQklnPon Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.SCz3hrheEF /tmp/tmp.X7FQklnPon ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n '' ']' + create_namespace cross-site-sharded-2508 + local namespace=cross-site-sharded-2508 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-2508' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-2508 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-2508 --ignore-not-found + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.izlF3Sj83u ++ mktemp + local LAST_ERR=/tmp/tmp.5JqxZodkFN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-sharded-2508 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.fLgK5ZKLLq ++ mktemp + local LAST_ERR=/tmp/tmp.fKLk3dBe4t + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fLgK5ZKLLq + cat /tmp/tmp.fKLk3dBe4t + rm /tmp/tmp.fLgK5ZKLLq /tmp/tmp.fKLk3dBe4t + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.izlF3Sj83u + cat /tmp/tmp.5JqxZodkFN + rm /tmp/tmp.izlF3Sj83u /tmp/tmp.5JqxZodkFN + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-2508 ++ mktemp + local LAST_OUT=/tmp/tmp.UUNWT2zMg7 ++ mktemp + local LAST_ERR=/tmp/tmp.C0SGeU7hJq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace cross-site-sharded-2508 namespace "cross-site-sharded-23632" deleted namespace "cross-site-sharded-replica-5989" deleted namespace "gmp-public" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UUNWT2zMg7 + cat /tmp/tmp.C0SGeU7hJq + rm /tmp/tmp.UUNWT2zMg7 /tmp/tmp.C0SGeU7hJq + return 0 + desc 'create namespace cross-site-sharded-2508' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-2508 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-2508 ++ mktemp + local LAST_OUT=/tmp/tmp.F4E1dcAaXY ++ mktemp + local LAST_ERR=/tmp/tmp.ZFzc3bB7fk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-sharded-2508 namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.F4E1dcAaXY namespace/cross-site-sharded-2508 created + cat /tmp/tmp.ZFzc3bB7fk + rm /tmp/tmp.F4E1dcAaXY /tmp/tmp.ZFzc3bB7fk + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.c0NRkdfSjB +++ mktemp ++ local LAST_ERR=/tmp/tmp.bLMmB5sX0V ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.c0NRkdfSjB ++ cat /tmp/tmp.bLMmB5sX0V ++ rm /tmp/tmp.c0NRkdfSjB /tmp/tmp.bLMmB5sX0V ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 ++ mktemp + local LAST_OUT=/tmp/tmp.Fi9mMej0Po ++ mktemp + local LAST_ERR=/tmp/tmp.HLZ3e2oDtg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Fi9mMej0Po Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.HLZ3e2oDtg + rm /tmp/tmp.Fi9mMej0Po /tmp/tmp.HLZ3e2oDtg + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.OTCGGWfIL5 ++ mktemp + local LAST_ERR=/tmp/tmp.NfmfOI7q2B + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OTCGGWfIL5 customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.NfmfOI7q2B + rm /tmp/tmp.OTCGGWfIL5 /tmp/tmp.NfmfOI7q2B + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.e6xC5D1TgN ++ mktemp + local LAST_ERR=/tmp/tmp.v5eAKpAaTs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + sed -e 's^namespace: .*^namespace: psmdb-operator^' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.e6xC5D1TgN role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.v5eAKpAaTs + rm /tmp/tmp.e6xC5D1TgN /tmp/tmp.v5eAKpAaTs + return 0 + kubectl_bin apply -f - + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1647-9959fb75") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ADbAQIbRd6 ++ mktemp + local LAST_ERR=/tmp/tmp.yNI50x1IwH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ADbAQIbRd6 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.yNI50x1IwH + rm /tmp/tmp.ADbAQIbRd6 /tmp/tmp.yNI50x1IwH + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pcMrCk0r7I +++ mktemp ++ local LAST_ERR=/tmp/tmp.LXemFKTAaX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pcMrCk0r7I ++ cat /tmp/tmp.LXemFKTAaX ++ rm /tmp/tmp.pcMrCk0r7I /tmp/tmp.LXemFKTAaX ++ return 0 + wait_pod percona-server-mongodb-operator-c95475bfb-5qcqk + local pod=percona-server-mongodb-operator-c95475bfb-5qcqk + set +o xtrace waiting for pod/percona-server-mongodb-operator-c95475bfb-5qcqk to be ready.OK + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xoSrPv7DYI ++ mktemp + local LAST_ERR=/tmp/tmp.z6UcGC24rR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xoSrPv7DYI deployment.apps/psmdb-client created secret/cross-site-sharded-main-secrets created secret/cross-site-sharded-main-ssl created secret/cross-site-sharded-main-ssl-internal created + cat /tmp/tmp.z6UcGC24rR + rm /tmp/tmp.xoSrPv7DYI /tmp/tmp.z6UcGC24rR + return 0 + desc 'create main PSMDB cluster cross-site-sharded-main.' + set +o xtrace ----------------------------------------------------------------------------------- create main PSMDB cluster cross-site-sharded-main. ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-main.yml ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1647-9959fb75"' + local LAST_OUT=/tmp/tmp.Om8AtYOXjl + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + local LAST_ERR=/tmp/tmp.IYSeglyjiP + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Om8AtYOXjl perconaservermongodb.psmdb.percona.com/cross-site-sharded-main created + cat /tmp/tmp.IYSeglyjiP + rm /tmp/tmp.Om8AtYOXjl /tmp/tmp.IYSeglyjiP + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-main-rs0 3 + local name=cross-site-sharded-main-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-0 + local pod=cross-site-sharded-main-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-0 to be ready...........OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-rs0-1 + local pod=cross-site-sharded-main-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-1 to be ready............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ArofK6lrU6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Jradv0IsOq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ArofK6lrU6 ++ cat /tmp/tmp.Jradv0IsOq ++ rm /tmp/tmp.ArofK6lrU6 /tmp/tmp.Jradv0IsOq ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-main-rs0-2 + local pod=cross-site-sharded-main-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-main-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.83PUkM95vD +++ mktemp ++ local LAST_ERR=/tmp/tmp.4qErYP7Lb8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.83PUkM95vD ++ cat /tmp/tmp.4qErYP7Lb8 ++ rm /tmp/tmp.83PUkM95vD /tmp/tmp.4qErYP7Lb8 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................................ + wait_for_running cross-site-sharded-main-cfg 3 false + local name=cross-site-sharded-main-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-main ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-0 + local pod=cross-site-sharded-main-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-main-cfg-1 + local pod=cross-site-sharded-main-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.riGMWzZ0mu +++ mktemp ++ local LAST_ERR=/tmp/tmp.pgdequ91DD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.riGMWzZ0mu ++ cat /tmp/tmp.pgdequ91DD ++ rm /tmp/tmp.riGMWzZ0mu /tmp/tmp.pgdequ91DD ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-main-cfg-2 + local pod=cross-site-sharded-main-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-main-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4FJ4DEdjNL +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zzc7Yih9HN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-main -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4FJ4DEdjNL ++ cat /tmp/tmp.Zzc7Yih9HN ++ rm /tmp/tmp.4FJ4DEdjNL /tmp/tmp.Zzc7Yih9HN ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yWYyS6SdRU +++ mktemp ++ local LAST_ERR=/tmp/tmp.DKY8Y4gTnL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yWYyS6SdRU ++ cat /tmp/tmp.DKY8Y4gTnL ++ rm /tmp/tmp.yWYyS6SdRU /tmp/tmp.DKY8Y4gTnL ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Bbj267TFXF ++ mktemp + local LAST_ERR=/tmp/tmp.BiVZEsDPi4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bbj267TFXF Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("aa0ecd31-ab6e-49ae-95e6-d3fccc1e7a14") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "user", "roles" : [ { "db" : "app", "role" : "readWrite" } ] } bye + cat /tmp/tmp.BiVZEsDPi4 + rm /tmp/tmp.Bbj267TFXF /tmp/tmp.BiVZEsDPi4 + return 0 + sleep 2 + desc 'set chunk size to 2 MB' + set +o xtrace ----------------------------------------------------------------------------------- set chunk size to 2 MB ----------------------------------------------------------------------------------- + run_mongos 'use config\n db.settings.save( { _id:"chunksize", value: 2 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=use config\n db.settings.save( { _id:"chunksize", value: 2 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TuJUQ05I0M +++ mktemp ++ local LAST_ERR=/tmp/tmp.G2ovjaMcVQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TuJUQ05I0M ++ cat /tmp/tmp.G2ovjaMcVQ ++ rm /tmp/tmp.TuJUQ05I0M /tmp/tmp.G2ovjaMcVQ ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.IiZegJ3UsB ++ mktemp + local LAST_ERR=/tmp/tmp.1gVjU0Qfhk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.settings.save( { _id:"chunksize", value: 2 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IiZegJ3UsB Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("f014d770-f985-43aa-9d1a-5aa2b60d6afa") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match switched to db config WriteResult({ "nMatched" : 0, "nUpserted" : 1, "nModified" : 0, "_id" : "chunksize" }) bye + cat /tmp/tmp.1gVjU0Qfhk + rm /tmp/tmp.IiZegJ3UsB /tmp/tmp.1gVjU0Qfhk + return 0 + sleep 2 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/data.js user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/data.js + local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p3hBwxIlhN +++ mktemp ++ local LAST_ERR=/tmp/tmp.d6GlrqXYjv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.p3hBwxIlhN ++ cat /tmp/tmp.d6GlrqXYjv ++ rm /tmp/tmp.p3hBwxIlhN /tmp/tmp.d6GlrqXYjv ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/data.js + name=data.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/data.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.Wd6YRRRKli ++ mktemp + local LAST_ERR=/tmp/tmp.pshZRiJhOq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/data.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Wd6YRRRKli + cat /tmp/tmp.pshZRiJhOq + rm /tmp/tmp.Wd6YRRRKli /tmp/tmp.pshZRiJhOq + return 0 + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin /tmp/data.js' ++ mktemp + local LAST_OUT=/tmp/tmp.fDpNT0Gt44 ++ mktemp + local LAST_ERR=/tmp/tmp.ihUtNsmyHQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin /tmp/data.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fDpNT0Gt44 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("e0aded4b-fa14-4f62-a5b9-b88e67fe17b5") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match + cat /tmp/tmp.ihUtNsmyHQ + rm /tmp/tmp.fDpNT0Gt44 /tmp/tmp.ihUtNsmyHQ + return 0 + desc 'shard collection' + set +o xtrace ----------------------------------------------------------------------------------- shard collection ----------------------------------------------------------------------------------- + run_mongos 'sh.enableSharding("app")' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=sh.enableSharding("app")' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HeBeENgBxw +++ mktemp ++ local LAST_ERR=/tmp/tmp.lNgimIWQuV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HeBeENgBxw ++ cat /tmp/tmp.lNgimIWQuV ++ rm /tmp/tmp.HeBeENgBxw /tmp/tmp.lNgimIWQuV ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.FpuFyRfRDU ++ mktemp + local LAST_ERR=/tmp/tmp.lgS688jvIn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''sh.enableSharding("app")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FpuFyRfRDU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("91910ca6-3a1c-4500-98fe-fbf8ce6114cb") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1727283222, 1), "signature" : { "hash" : BinData(0,"q75LHZvwm4cGYUzfmmvdPA0r8mI="), "keyId" : NumberLong("7418624094721015821") } }, "operationTime" : Timestamp(1727283222, 1) } bye + cat /tmp/tmp.lgS688jvIn + rm /tmp/tmp.FpuFyRfRDU /tmp/tmp.lgS688jvIn + return 0 + sleep 2 + run_mongos 'sh.shardCollection("app.city", { _id: 1 } )' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=sh.shardCollection("app.city", { _id: 1 } )' + local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.J2zdNkj1Im +++ mktemp ++ local LAST_ERR=/tmp/tmp.b5NhX22Sjr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.J2zdNkj1Im ++ cat /tmp/tmp.b5NhX22Sjr ++ rm /tmp/tmp.J2zdNkj1Im /tmp/tmp.b5NhX22Sjr ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.E4mZryXKui ++ mktemp + local LAST_ERR=/tmp/tmp.VgUeMtOHbf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''sh.shardCollection("app.city", { _id: 1 } )\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E4mZryXKui Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("d4cffbff-5ec7-434d-9155-6755b1d7dd4f") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match { "collectionsharded" : "app.city", "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1727283228, 35), "signature" : { "hash" : BinData(0,"Y3iXDphL0ZjIT6hjBc0I5qG3ZvI="), "keyId" : NumberLong("7418624094721015821") } }, "operationTime" : Timestamp(1727283228, 35) } bye + cat /tmp/tmp.VgUeMtOHbf + rm /tmp/tmp.E4mZryXKui /tmp/tmp.VgUeMtOHbf + return 0 + sleep 120 + desc 'Check chunks' + set +o xtrace ----------------------------------------------------------------------------------- Check chunks ----------------------------------------------------------------------------------- + chunks_param1=ns + chunks_param2='"app.city"' + [[ 7.0 != \4\.\4 ]] + chunks_param1=uuid ++ run_mongos 'use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ local 'command=use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid' ++ grep 'switched to db app' -A 1 ++ local uri=user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ grep -v 'switched to db app' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZXu3pH8bAJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.odCYXiW170 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ZXu3pH8bAJ +++ cat /tmp/tmp.odCYXiW170 +++ rm /tmp/tmp.ZXu3pH8bAJ /tmp/tmp.odCYXiW170 +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SvMcWO3rqc +++ mktemp ++ local LAST_ERR=/tmp/tmp.PZmPe4kOcH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use app\n db.getCollectionInfos({ "name": "city" })[0].info.uuid\n'\'' | mongo mongodb://user:pass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.SvMcWO3rqc ++ cat /tmp/tmp.PZmPe4kOcH ++ rm /tmp/tmp.SvMcWO3rqc /tmp/tmp.PZmPe4kOcH ++ return 0 + chunks_param2='UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6")' + shards=0 + for i in '"rs0"' '"rs1"' ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs0"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ grep 'switched to db config' -A 1 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs0"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ local driver=mongodb ++ local suffix=.svc.cluster.local ++ grep -v 'switched to db config' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.a4eQ07Ae2Q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V9aAGGlEW8 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.a4eQ07Ae2Q +++ cat /tmp/tmp.V9aAGGlEW8 +++ rm /tmp/tmp.a4eQ07Ae2Q /tmp/tmp.V9aAGGlEW8 +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.araH6fmQtw +++ mktemp ++ local LAST_ERR=/tmp/tmp.hyc1wVPL5F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs0"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.araH6fmQtw ++ cat /tmp/tmp.hyc1wVPL5F ++ rm /tmp/tmp.araH6fmQtw /tmp/tmp.hyc1wVPL5F ++ return 0 + out=1 + desc 'rs0 has 1 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs0 has 1 chunks ----------------------------------------------------------------------------------- + [[ 1 -ne 0 ]] + (( shards = shards + 1 )) + for i in '"rs0"' '"rs1"' ++ run_mongos 'use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs1"})' clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ grep 'switched to db config' -A 1 ++ local 'command=use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs1"})' ++ local uri=clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 ++ local driver=mongodb ++ local suffix=.svc.cluster.local +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp ++ grep -v 'switched to db config' +++ local LAST_OUT=/tmp/tmp.43ksWjBC6r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8QEUNQ6lFl +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.43ksWjBC6r +++ cat /tmp/tmp.8QEUNQ6lFl +++ rm /tmp/tmp.43ksWjBC6r /tmp/tmp.8QEUNQ6lFl +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cv5FcCC83x +++ mktemp ++ local LAST_ERR=/tmp/tmp.VECw3sA20M ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''use config\n db.chunks.count({"uuid": UUID("d3e88ea6-3710-483b-b604-c90d2a1329f6"), "shard": "rs1"})\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cv5FcCC83x ++ cat /tmp/tmp.VECw3sA20M ++ rm /tmp/tmp.cv5FcCC83x /tmp/tmp.VECw3sA20M ++ return 0 + out=3 + desc 'rs1 has 3 chunks' + set +o xtrace ----------------------------------------------------------------------------------- rs1 has 3 chunks ----------------------------------------------------------------------------------- + [[ 3 -ne 0 ]] + (( shards = shards + 1 )) + [[ 2 -lt 2 ]] + desc 'create replica cluster' + set +o xtrace ----------------------------------------------------------------------------------- create replica cluster ----------------------------------------------------------------------------------- + create_namespace cross-site-sharded-replica-13364 0 + local namespace=cross-site-sharded-replica-13364 + local skip_clean_namespace=0 + [[ 1 == 1 ]] + [[ -z 0 ]] + '[' -n '' ']' + desc 'cleaned up old namespaces cross-site-sharded-replica-13364' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces cross-site-sharded-replica-13364 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace cross-site-sharded-replica-13364 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.l28aECUFa8 ++ mktemp + local LAST_ERR=/tmp/tmp.jjx3Jjsb3D + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace cross-site-sharded-replica-13364 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l28aECUFa8 + cat /tmp/tmp.jjx3Jjsb3D + rm /tmp/tmp.l28aECUFa8 /tmp/tmp.jjx3Jjsb3D + return 0 + kubectl_bin wait --for=delete namespace cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.tud9DVk61X ++ mktemp + local LAST_ERR=/tmp/tmp.IJ68h7PlBR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace cross-site-sharded-replica-13364 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tud9DVk61X + cat /tmp/tmp.IJ68h7PlBR + rm /tmp/tmp.tud9DVk61X /tmp/tmp.IJ68h7PlBR + return 0 + desc 'create namespace cross-site-sharded-replica-13364' + set +o xtrace ----------------------------------------------------------------------------------- create namespace cross-site-sharded-replica-13364 ----------------------------------------------------------------------------------- + kubectl_bin create namespace cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.dLp7YNDMLq ++ mktemp + local LAST_ERR=/tmp/tmp.IH17VYGCqz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cross-site-sharded-replica-13364 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dLp7YNDMLq namespace/cross-site-sharded-replica-13364 created + cat /tmp/tmp.IH17VYGCqz + rm /tmp/tmp.dLp7YNDMLq /tmp/tmp.IH17VYGCqz + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.8KFs86NdIm +++ mktemp ++ local LAST_ERR=/tmp/tmp.6jkXO1CA3Y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8KFs86NdIm ++ cat /tmp/tmp.6jkXO1CA3Y ++ rm /tmp/tmp.8KFs86NdIm /tmp/tmp.6jkXO1CA3Y ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.X5kxJCwPVI ++ mktemp + local LAST_ERR=/tmp/tmp.gr2ibaeS6x + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.X5kxJCwPVI Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.gr2ibaeS6x + rm /tmp/tmp.X5kxJCwPVI /tmp/tmp.gr2ibaeS6x + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.lLmsq7HONs ++ mktemp + local LAST_ERR=/tmp/tmp.d0saUWjDHx + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lLmsq7HONs customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.d0saUWjDHx + rm /tmp/tmp.lLmsq7HONs /tmp/tmp.d0saUWjDHx + return 0 + '[' -n '' ']' + apply_rbac rbac + local operator_namespace=psmdb-operator + local rbac=rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AxsCg8lGpV ++ mktemp + local LAST_ERR=/tmp/tmp.dRkAMMVlsc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AxsCg8lGpV role.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created rolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.dRkAMMVlsc + rm /tmp/tmp.AxsCg8lGpV /tmp/tmp.dRkAMMVlsc + return 0 + kubectl_bin apply -f - + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1647-9959fb75") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.IXIeS9IokJ ++ mktemp + local LAST_ERR=/tmp/tmp.mAvqIqEq2P + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IXIeS9IokJ deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.mAvqIqEq2P + rm /tmp/tmp.IXIeS9IokJ /tmp/tmp.mAvqIqEq2P + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9SvHNF4dMm +++ mktemp ++ local LAST_ERR=/tmp/tmp.lNsdAstshY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9SvHNF4dMm ++ cat /tmp/tmp.lNsdAstshY ++ rm /tmp/tmp.9SvHNF4dMm /tmp/tmp.lNsdAstshY ++ return 0 + wait_pod percona-server-mongodb-operator-c95475bfb-hbv57 + local pod=percona-server-mongodb-operator-c95475bfb-hbv57 + set +o xtrace waiting for pod/percona-server-mongodb-operator-c95475bfb-hbv57 to be ready.OK + desc 'start client' + set +o xtrace ----------------------------------------------------------------------------------- start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.U70m6q6hcJ ++ mktemp + local LAST_ERR=/tmp/tmp.jV6f0nna20 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U70m6q6hcJ deployment.apps/psmdb-client created + cat /tmp/tmp.jV6f0nna20 + rm /tmp/tmp.U70m6q6hcJ /tmp/tmp.jV6f0nna20 + return 0 + desc 'copy secrets from main to replica namespace and create all of them' + set +o xtrace ----------------------------------------------------------------------------------- copy secrets from main to replica namespace and create all of them ----------------------------------------------------------------------------------- + kubectl get secret cross-site-sharded-main-secrets -o yaml -n cross-site-sharded-2508 + yq eval ' del(.metadata) | (.metadata.name = "cross-site-sharded-replica-secrets")' - + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.TNHbTqxbdY ++ mktemp + local LAST_ERR=/tmp/tmp.5YABUkOpPq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TNHbTqxbdY secret/cross-site-sharded-replica-secrets created + cat /tmp/tmp.5YABUkOpPq + rm /tmp/tmp.TNHbTqxbdY /tmp/tmp.5YABUkOpPq + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-2508 + kubectl_bin apply -f - + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl-internal")' - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.JH6Rn2PUBQ ++ mktemp + local LAST_OUT=/tmp/tmp.dm4QBIZhg4 + local LAST_ERR=/tmp/tmp.NMJpoLQDiX + local exit_status=0 ++ mktemp + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.mcDfLKmbhF + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secret cross-site-sharded-main-ssl-internal -o yaml -n cross-site-sharded-2508 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JH6Rn2PUBQ + cat /tmp/tmp.NMJpoLQDiX + rm /tmp/tmp.JH6Rn2PUBQ /tmp/tmp.NMJpoLQDiX + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.dm4QBIZhg4 secret/cross-site-sharded-replica-ssl-internal created + cat /tmp/tmp.mcDfLKmbhF + rm /tmp/tmp.dm4QBIZhg4 /tmp/tmp.mcDfLKmbhF + return 0 + kubectl_bin get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-2508 + yq eval ' del(.metadata) | del(.status) | (.metadata.name = "cross-site-sharded-replica-ssl")' - + kubectl_bin apply -f - ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.oHguwnN6yK + local LAST_OUT=/tmp/tmp.07j1ojXYZM ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.YM4UMJCD1q + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.rGJRI6LVfn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get secret cross-site-sharded-main-ssl -o yaml -n cross-site-sharded-2508 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oHguwnN6yK + cat /tmp/tmp.YM4UMJCD1q + rm /tmp/tmp.oHguwnN6yK /tmp/tmp.YM4UMJCD1q + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.07j1ojXYZM secret/cross-site-sharded-replica-ssl created + cat /tmp/tmp.rGJRI6LVfn + rm /tmp/tmp.07j1ojXYZM /tmp/tmp.rGJRI6LVfn + return 0 + sleep 30 + desc 'create replica PSMDB cluster ' + set +o xtrace ----------------------------------------------------------------------------------- create replica PSMDB cluster ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/conf/cross-site-sharded-replica.yml + local LAST_OUT=/tmp/tmp.yImKZuEeJE + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '.spec.upgradeOptions.apply="Never"' + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_ERR=/tmp/tmp.0RGRDVE8OQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1647-9959fb75"' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yImKZuEeJE perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica created + cat /tmp/tmp.0RGRDVE8OQ + rm /tmp/tmp.yImKZuEeJE /tmp/tmp.0RGRDVE8OQ + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 false + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.............OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.............OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.frm6yIURuD +++ mktemp ++ local LAST_ERR=/tmp/tmp.z1IGN8iKfs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.frm6yIURuD ++ cat /tmp/tmp.z1IGN8iKfs ++ rm /tmp/tmp.frm6yIURuD /tmp/tmp.z1IGN8iKfs ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready...........OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qfVLCr4h7U +++ mktemp ++ local LAST_ERR=/tmp/tmp.bRxa1HbIOu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qfVLCr4h7U ++ cat /tmp/tmp.bRxa1HbIOu ++ rm /tmp/tmp.qfVLCr4h7U /tmp/tmp.bRxa1HbIOu ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-rs1 3 false + local name=cross-site-sharded-replica-rs1 + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs1 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-0 + local pod=cross-site-sharded-replica-rs1-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs1-1 + local pod=cross-site-sharded-replica-rs1-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZsisYyVeTK +++ mktemp ++ local LAST_ERR=/tmp/tmp.fWb8aMsbR2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZsisYyVeTK ++ cat /tmp/tmp.fWb8aMsbR2 ++ rm /tmp/tmp.ZsisYyVeTK /tmp/tmp.fWb8aMsbR2 ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs1-2 + local pod=cross-site-sharded-replica-rs1-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs1-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xjntx7VMPX +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gq1WzexhjH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs1")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.xjntx7VMPX ++ cat /tmp/tmp.Gq1WzexhjH ++ rm /tmp/tmp.xjntx7VMPX /tmp/tmp.Gq1WzexhjH ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bl03Fxm9gC +++ mktemp ++ local LAST_ERR=/tmp/tmp.av60vhvqtm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bl03Fxm9gC ++ cat /tmp/tmp.av60vhvqtm ++ rm /tmp/tmp.bl03Fxm9gC /tmp/tmp.av60vhvqtm ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2zpCNLs7zz +++ mktemp ++ local LAST_ERR=/tmp/tmp.BBvXXRlxuZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2zpCNLs7zz ++ cat /tmp/tmp.BBvXXRlxuZ ++ rm /tmp/tmp.2zpCNLs7zz /tmp/tmp.BBvXXRlxuZ ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] ++ get_service_ip cross-site-sharded-replica-cfg-0 cfg ++ local service=cross-site-sharded-replica-cfg-0 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TPy3ee0FRf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EqadC7Kp8Q +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.TPy3ee0FRf +++ cat /tmp/tmp.EqadC7Kp8Q +++ rm /tmp/tmp.TPy3ee0FRf /tmp/tmp.EqadC7Kp8Q +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xejpK9b5iM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.P3wZQEQVeZ +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.xejpK9b5iM +++ cat /tmp/tmp.P3wZQEQVeZ +++ rm /tmp/tmp.xejpK9b5iM /tmp/tmp.P3wZQEQVeZ +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VBHEliysrI +++ mktemp ++ local LAST_ERR=/tmp/tmp.EVHHqNHHap ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VBHEliysrI ++ cat /tmp/tmp.EVHHqNHHap ++ rm /tmp/tmp.VBHEliysrI /tmp/tmp.EVHHqNHHap ++ return 0 ++ return + replica_cfg_0_endpoint=10.36.41.154 ++ get_service_ip cross-site-sharded-replica-cfg-1 cfg ++ local service=cross-site-sharded-replica-cfg-1 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mXc5Quk48L ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XlUetXgN7m +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.mXc5Quk48L +++ cat /tmp/tmp.XlUetXgN7m +++ rm /tmp/tmp.mXc5Quk48L /tmp/tmp.XlUetXgN7m +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dVymUSiPfC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KEMXtMI1Gi +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dVymUSiPfC +++ cat /tmp/tmp.KEMXtMI1Gi +++ rm /tmp/tmp.dVymUSiPfC /tmp/tmp.KEMXtMI1Gi +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.II3sIduw9e +++ mktemp ++ local LAST_ERR=/tmp/tmp.qat5UlqWyT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.II3sIduw9e ++ cat /tmp/tmp.qat5UlqWyT ++ rm /tmp/tmp.II3sIduw9e /tmp/tmp.qat5UlqWyT ++ return 0 ++ return + replica_cfg_1_endpoint=10.36.40.146 ++ get_service_ip cross-site-sharded-replica-cfg-2 cfg ++ local service=cross-site-sharded-replica-cfg-2 ++ local server_type=cfg +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vOv58jFcdw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IcogKPkTYU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.vOv58jFcdw +++ cat /tmp/tmp.IcogKPkTYU +++ rm /tmp/tmp.vOv58jFcdw /tmp/tmp.IcogKPkTYU +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jnS9Q6de1E ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9xOdzn3CBU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.jnS9Q6de1E +++ cat /tmp/tmp.9xOdzn3CBU +++ rm /tmp/tmp.jnS9Q6de1E /tmp/tmp.9xOdzn3CBU +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.y0M0UvfWv4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lQ8TeOBvxI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-cfg-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.y0M0UvfWv4 ++ cat /tmp/tmp.lQ8TeOBvxI ++ rm /tmp/tmp.y0M0UvfWv4 /tmp/tmp.lQ8TeOBvxI ++ return 0 ++ return + replica_cfg_2_endpoint=10.36.35.70 ++ get_service_ip cross-site-sharded-replica-rs0-0 ++ local service=cross-site-sharded-replica-rs0-0 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Aoh8yOupK8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oMxh5M7Wkr +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.Aoh8yOupK8 +++ cat /tmp/tmp.oMxh5M7Wkr +++ rm /tmp/tmp.Aoh8yOupK8 /tmp/tmp.oMxh5M7Wkr +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ErZCmGzy49 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kZKa7AQjcY +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.ErZCmGzy49 +++ cat /tmp/tmp.kZKa7AQjcY +++ rm /tmp/tmp.ErZCmGzy49 /tmp/tmp.kZKa7AQjcY +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.r3C84Qmmca +++ mktemp ++ local LAST_ERR=/tmp/tmp.OsztPXrs15 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.r3C84Qmmca ++ cat /tmp/tmp.OsztPXrs15 ++ rm /tmp/tmp.r3C84Qmmca /tmp/tmp.OsztPXrs15 ++ return 0 ++ return + replica_rs0_0_endpoint=10.36.44.28 ++ get_service_ip cross-site-sharded-replica-rs0-1 ++ local service=cross-site-sharded-replica-rs0-1 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5I7g0pvYFF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7KBfyepMya +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.5I7g0pvYFF +++ cat /tmp/tmp.7KBfyepMya +++ rm /tmp/tmp.5I7g0pvYFF /tmp/tmp.7KBfyepMya +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1roNirRSPS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.whMmr4ybZw +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.1roNirRSPS +++ cat /tmp/tmp.whMmr4ybZw +++ rm /tmp/tmp.1roNirRSPS /tmp/tmp.whMmr4ybZw +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nNWo9uFOv2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EGCNVNTyvD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nNWo9uFOv2 ++ cat /tmp/tmp.EGCNVNTyvD ++ rm /tmp/tmp.nNWo9uFOv2 /tmp/tmp.EGCNVNTyvD ++ return 0 ++ return + replica_rs0_1_endpoint=10.36.41.10 ++ get_service_ip cross-site-sharded-replica-rs0-2 ++ local service=cross-site-sharded-replica-rs0-2 ++ local server_type=rs0 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CD2A7ohm4L ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kMcWHDnCf9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.CD2A7ohm4L +++ cat /tmp/tmp.kMcWHDnCf9 +++ rm /tmp/tmp.CD2A7ohm4L /tmp/tmp.kMcWHDnCf9 +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zaQynzFkjj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M0ShFiayjl +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.zaQynzFkjj +++ cat /tmp/tmp.M0ShFiayjl +++ rm /tmp/tmp.zaQynzFkjj /tmp/tmp.M0ShFiayjl +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VX8LS6nLsS +++ mktemp ++ local LAST_ERR=/tmp/tmp.biypYTgNTM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs0-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VX8LS6nLsS ++ cat /tmp/tmp.biypYTgNTM ++ rm /tmp/tmp.VX8LS6nLsS /tmp/tmp.biypYTgNTM ++ return 0 ++ return + replica_rs0_2_endpoint=10.36.39.80 ++ get_service_ip cross-site-sharded-replica-rs1-0 rs1 ++ local service=cross-site-sharded-replica-rs1-0 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gnVeUQKrAV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HsGl8Gq8SO +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.gnVeUQKrAV +++ cat /tmp/tmp.HsGl8Gq8SO +++ rm /tmp/tmp.gnVeUQKrAV /tmp/tmp.HsGl8Gq8SO +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DfwdEvQMvq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W3TuVvrkze +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.DfwdEvQMvq +++ cat /tmp/tmp.W3TuVvrkze +++ rm /tmp/tmp.DfwdEvQMvq /tmp/tmp.W3TuVvrkze +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jonm0MlAxx +++ mktemp ++ local LAST_ERR=/tmp/tmp.b1DaJ2OqqB ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-0 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.jonm0MlAxx ++ cat /tmp/tmp.b1DaJ2OqqB ++ rm /tmp/tmp.jonm0MlAxx /tmp/tmp.b1DaJ2OqqB ++ return 0 ++ return + replica_rs1_0_endpoint=10.36.38.32 ++ get_service_ip cross-site-sharded-replica-rs1-1 rs1 ++ local service=cross-site-sharded-replica-rs1-1 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.33rMKsWTOU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1ukQV6IhL9 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.33rMKsWTOU +++ cat /tmp/tmp.1ukQV6IhL9 +++ rm /tmp/tmp.33rMKsWTOU /tmp/tmp.1ukQV6IhL9 +++ return 0 ++ '[' true '!=' true ']' ++ grep -q NotFound ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AKz8rjord1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sHXzAN5TIK +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.AKz8rjord1 +++ cat /tmp/tmp.sHXzAN5TIK +++ rm /tmp/tmp.AKz8rjord1 /tmp/tmp.sHXzAN5TIK +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.C35MN8hlOI +++ mktemp ++ local LAST_ERR=/tmp/tmp.oKZFhBWLMk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-1 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.C35MN8hlOI ++ cat /tmp/tmp.oKZFhBWLMk ++ rm /tmp/tmp.C35MN8hlOI /tmp/tmp.oKZFhBWLMk ++ return 0 ++ return + replica_rs1_1_endpoint=10.36.39.243 ++ get_service_ip cross-site-sharded-replica-rs1-2 rs1 ++ local service=cross-site-sharded-replica-rs1-2 ++ local server_type=rs1 +++ kubectl_bin get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dY769qZu1U ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Y5dn3CFg6N +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get psmdb/cross-site-sharded-replica -o 'jsonpath={.spec.replsets[].expose.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.dY769qZu1U +++ cat /tmp/tmp.Y5dn3CFg6N +++ rm /tmp/tmp.dY769qZu1U /tmp/tmp.Y5dn3CFg6N +++ return 0 ++ '[' true '!=' true ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++ grep -q NotFound +++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oZQcbGozmy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UuTTUK75uo +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.type}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.oZQcbGozmy +++ cat /tmp/tmp.UuTTUK75uo +++ rm /tmp/tmp.oZQcbGozmy /tmp/tmp.UuTTUK75uo +++ return 0 ++ service_type=ClusterIP ++ '[' ClusterIP = ClusterIP ']' ++ kubectl_bin get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.plwboPMfD3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.FzixfIQMxq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get service/cross-site-sharded-replica-rs1-2 -o 'jsonpath={.spec.clusterIP}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.plwboPMfD3 ++ cat /tmp/tmp.FzixfIQMxq ++ rm /tmp/tmp.plwboPMfD3 /tmp/tmp.FzixfIQMxq ++ return 0 ++ return + replica_rs1_2_endpoint=10.36.33.153 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.PlK2QvlACI +++ mktemp ++ local LAST_ERR=/tmp/tmp.U8MMpNhVFN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PlK2QvlACI ++ cat /tmp/tmp.U8MMpNhVFN ++ rm /tmp/tmp.PlK2QvlACI /tmp/tmp.U8MMpNhVFN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 ++ mktemp + local LAST_OUT=/tmp/tmp.SHrs7iizli ++ mktemp + local LAST_ERR=/tmp/tmp.DjmohMzcIZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SHrs7iizli Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.DjmohMzcIZ + rm /tmp/tmp.SHrs7iizli /tmp/tmp.DjmohMzcIZ + return 0 + kubectl_bin patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"10.36.44.28","priority":0,"votes":0},{"host":"10.36.41.10","port":27017,"priority":1,"votes":1},{"host":"10.36.39.80", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"10.36.38.32","priority":0,"votes":0},{"host":"10.36.39.243","port":27017,"priority":1,"votes":1},{"host":"10.36.33.153", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.36.41.154","priority":1,"votes":1 },{"host":"10.36.40.146", "priority":1,"votes":1},{"host":"10.36.35.70","priority":0,"votes":0}]}} } }' ++ mktemp + local LAST_OUT=/tmp/tmp.kk8F9HYo6j ++ mktemp + local LAST_ERR=/tmp/tmp.9Q7s5gP2Ma + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch psmdb cross-site-sharded-main --type=merge --patch '{ "spec": {"replsets":[ {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"10.36.44.28","priority":0,"votes":0},{"host":"10.36.41.10","port":27017,"priority":1,"votes":1},{"host":"10.36.39.80", "port":27017,"priority":1,"votes":1}],"name":"rs0","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}, {"affinity":{"antiAffinityTopologyKey": "none"},"arbiter":{"affinity":{"antiAffinityTopologyKey": "none"},"enabled":false,"size":1},"expose":{"enabled":true,"type":"ClusterIp"},"externalNodes":[{"host":"10.36.38.32","priority":0,"votes":0},{"host":"10.36.39.243","port":27017,"priority":1,"votes":1},{"host":"10.36.33.153", "port":27017,"priority":1,"votes":1}],"name":"rs1","nonvoting":{"affinity":{"antiAffinityTopologyKey":"none"},"enabled":false,"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"1Gi"}}}}},"podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"cpu":"300m","memory":"0.5G"},"requests":{"cpu":"300m","memory":"0.5G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}} ], "sharding":{"configsvrReplSet":{ "externalNodes": [{"host":"10.36.41.154","priority":1,"votes":1 },{"host":"10.36.40.146", "priority":1,"votes":1},{"host":"10.36.35.70","priority":0,"votes":0}]}} } }' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kk8F9HYo6j perconaservermongodb.psmdb.percona.com/cross-site-sharded-main patched + cat /tmp/tmp.9Q7s5gP2Ma + rm /tmp/tmp.kk8F9HYo6j /tmp/tmp.9Q7s5gP2Ma + return 0 + wait_for_members 10.36.41.154 cfg + local endpoint=10.36.41.154 + local rsName=cfg + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.36.41.154 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.36.41.154 ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rsTmlpkWWy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1posu04Hz0 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.rsTmlpkWWy +++ cat /tmp/tmp.1posu04Hz0 +++ rm /tmp/tmp.rsTmlpkWWy /tmp/tmp.1posu04Hz0 +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FCUTuCCAdw +++ mktemp ++ local LAST_ERR=/tmp/tmp.E7XcpCd93l ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FCUTuCCAdw ++ cat /tmp/tmp.E7XcpCd93l ++ rm /tmp/tmp.FCUTuCCAdw /tmp/tmp.E7XcpCd93l ++ return 0 + nodes_amount=4 + echo 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg + let retry+=1 + '[' 1 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 4 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.36.41.154 mongodb :27017 ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.36.41.154 ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QlHEO0z3oy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EOX4PLcIGU +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.QlHEO0z3oy +++ cat /tmp/tmp.EOX4PLcIGU +++ rm /tmp/tmp.QlHEO0z3oy /tmp/tmp.EOX4PLcIGU +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ug4JJr7hOX +++ mktemp ++ local LAST_ERR=/tmp/tmp.mujm7yMLOp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Ug4JJr7hOX ++ cat /tmp/tmp.mujm7yMLOp ++ rm /tmp/tmp.Ug4JJr7hOX /tmp/tmp.mujm7yMLOp ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in cfg' waiting for all members to be configured in cfg + let retry+=1 + '[' 2 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] + wait_for_members 10.36.44.28 rs0 + local endpoint=10.36.44.28 + local rsName=rs0 + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.36.44.28 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++ local uri=clusterAdmin:clusterAdmin123456@10.36.44.28 ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n3WpNN2QIl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dGlF4GZxcx +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.n3WpNN2QIl +++ cat /tmp/tmp.dGlF4GZxcx +++ rm /tmp/tmp.n3WpNN2QIl /tmp/tmp.dGlF4GZxcx +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.44.28:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d1dKvZbQr2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.EblCLXeKbW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.44.28:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d1dKvZbQr2 ++ cat /tmp/tmp.EblCLXeKbW ++ rm /tmp/tmp.d1dKvZbQr2 /tmp/tmp.EblCLXeKbW ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in rs0' waiting for all members to be configured in rs0 + let retry+=1 + '[' 3 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] + wait_for_members 10.36.38.32 rs1 + local endpoint=10.36.38.32 + local rsName=rs1 + local nodes_amount=0 + [[ 0 == 6 ]] ++ run_mongos 'rs.conf().members.length' clusterAdmin:clusterAdmin123456@10.36.38.32 mongodb :27017 ++ local 'command=rs.conf().members.length' ++ local uri=clusterAdmin:clusterAdmin123456@10.36.38.32 ++ local driver=mongodb ++ local suffix=:27017 ++ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PzX4FB1TjO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OtExtFIk57 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.PzX4FB1TjO +++ cat /tmp/tmp.OtExtFIk57 +++ rm /tmp/tmp.PzX4FB1TjO /tmp/tmp.OtExtFIk57 +++ return 0 ++ local client_container=psmdb-client-6c585f8dbd-hs4lz ++ local mongo_flag= ++ kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.38.32:27017/admin ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2ZP5AL9U6i +++ mktemp ++ local LAST_ERR=/tmp/tmp.t5SRCisoCM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'printf '\''rs.conf().members.length\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.38.32:27017/admin ' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2ZP5AL9U6i ++ cat /tmp/tmp.t5SRCisoCM ++ rm /tmp/tmp.2ZP5AL9U6i /tmp/tmp.t5SRCisoCM ++ return 0 + nodes_amount=6 + echo 'waiting for all members to be configured in rs1' waiting for all members to be configured in rs1 + let retry+=1 + '[' 4 -ge 15 ']' + echo -n . .+ sleep 10 + [[ 6 == 6 ]] ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.FH3djcYpIc +++ mktemp ++ local LAST_ERR=/tmp/tmp.G1HMz1bJfa ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FH3djcYpIc ++ cat /tmp/tmp.G1HMz1bJfa ++ rm /tmp/tmp.FH3djcYpIc /tmp/tmp.G1HMz1bJfa ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.ssxeRpGIaj ++ mktemp + local LAST_ERR=/tmp/tmp.o0T4rrHdYp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ssxeRpGIaj Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.o0T4rrHdYp + rm /tmp/tmp.ssxeRpGIaj /tmp/tmp.o0T4rrHdYp + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dWUJ2Rle9u +++ mktemp ++ local LAST_ERR=/tmp/tmp.pA46P1TYNZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dWUJ2Rle9u ++ cat /tmp/tmp.pA46P1TYNZ ++ rm /tmp/tmp.dWUJ2Rle9u /tmp/tmp.pA46P1TYNZ ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.US1bKBfx7q +++ mktemp ++ local LAST_ERR=/tmp/tmp.dsc9MCdmXH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.US1bKBfx7q ++ cat /tmp/tmp.dsc9MCdmXH ++ rm /tmp/tmp.US1bKBfx7q /tmp/tmp.dsc9MCdmXH ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + wait_for_running cross-site-sharded-replica-cfg 3 false + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oFlAvVDjDZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.GJvUeQIocF ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oFlAvVDjDZ ++ cat /tmp/tmp.GJvUeQIocF ++ rm /tmp/tmp.oFlAvVDjDZ /tmp/tmp.GJvUeQIocF ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0A54RS3QXI +++ mktemp ++ local LAST_ERR=/tmp/tmp.IA1egzl3AV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0A54RS3QXI ++ cat /tmp/tmp.IA1egzl3AV ++ rm /tmp/tmp.0A54RS3QXI /tmp/tmp.IA1egzl3AV ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + desc 'create user' + set +o xtrace ----------------------------------------------------------------------------------- create user ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KmBVVcbfCI +++ mktemp ++ local LAST_ERR=/tmp/tmp.xCd07b3HBw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KmBVVcbfCI ++ cat /tmp/tmp.xCd07b3HBw ++ rm /tmp/tmp.KmBVVcbfCI /tmp/tmp.xCd07b3HBw ++ return 0 + local client_container=psmdb-client-6c585f8dbd-9x4mz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.I8KXCBUPxZ ++ mktemp + local LAST_ERR=/tmp/tmp.tcJqoFEZr5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I8KXCBUPxZ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("bae19a29-7f36-49ea-87bb-37c16d150bdd") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.tcJqoFEZr5 + rm /tmp/tmp.I8KXCBUPxZ /tmp/tmp.tcJqoFEZr5 + return 0 + sleep 2 + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kEG53VM9Ne +++ mktemp ++ local LAST_ERR=/tmp/tmp.gLdpo64dGh ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kEG53VM9Ne ++ cat /tmp/tmp.gLdpo64dGh ++ rm /tmp/tmp.kEG53VM9Ne /tmp/tmp.gLdpo64dGh ++ return 0 + local client_container=psmdb-client-6c585f8dbd-9x4mz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.jOWRCsUOmY ++ mktemp + local LAST_ERR=/tmp/tmp.uZy8HT1Bvd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jOWRCsUOmY Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("8f0bf06b-a1cb-4b14-9906-41ec2ff6e49a") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.uZy8HT1Bvd + rm /tmp/tmp.jOWRCsUOmY /tmp/tmp.uZy8HT1Bvd + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + desc 'Compare data' + set +o xtrace ----------------------------------------------------------------------------------- Compare data ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local command=find + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local postfix= + local suffix= + local database=myApp + local collection=test + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.d0i8c20jHI +++ mktemp ++ local LAST_ERR=/tmp/tmp.NpqQI7rdlG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.d0i8c20jHI ++ cat /tmp/tmp.NpqQI7rdlG ++ rm /tmp/tmp.d0i8c20jHI /tmp/tmp.NpqQI7rdlG ++ return 0 + local client_container=psmdb-client-6c585f8dbd-9x4mz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.H41yHJIiFQ ++ mktemp + local LAST_ERR=/tmp/tmp.0uvSB06ERh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-main-mongos.cross-site-sharded-2508.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H41yHJIiFQ + cat /tmp/tmp.0uvSB06ERh + rm /tmp/tmp.H41yHJIiFQ /tmp/tmp.0uvSB06ERh + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.IhVzFBGGe8/find + desc 'test failover' + set +o xtrace ----------------------------------------------------------------------------------- test failover ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.HxsdZpJNgv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Depsu5X2Z0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HxsdZpJNgv ++ cat /tmp/tmp.Depsu5X2Z0 ++ rm /tmp/tmp.HxsdZpJNgv /tmp/tmp.Depsu5X2Z0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 ++ mktemp + local LAST_OUT=/tmp/tmp.k5sk4S9LHN ++ mktemp + local LAST_ERR=/tmp/tmp.6droYnXhwj + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-2508 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.k5sk4S9LHN Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.6droYnXhwj + rm /tmp/tmp.k5sk4S9LHN /tmp/tmp.6droYnXhwj + return 0 + kubectl_bin delete psmdb cross-site-sharded-main ++ mktemp + local LAST_OUT=/tmp/tmp.iTIQaB0d4a ++ mktemp + local LAST_ERR=/tmp/tmp.skodX6czag + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete psmdb cross-site-sharded-main + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iTIQaB0d4a perconaservermongodb.psmdb.percona.com "cross-site-sharded-main" deleted + cat /tmp/tmp.skodX6czag + rm /tmp/tmp.iTIQaB0d4a /tmp/tmp.skodX6czag + return 0 + desc 'run disaster recovery script for replset: cfg' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: cfg ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.36.41.154 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.36.41.154 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zMnif8PhYT +++ mktemp ++ local LAST_ERR=/tmp/tmp.kLoXUZj1V2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zMnif8PhYT ++ cat /tmp/tmp.kLoXUZj1V2 ++ rm /tmp/tmp.zMnif8PhYT /tmp/tmp.kLoXUZj1V2 ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.gLy04SbQNR ++ mktemp + local LAST_ERR=/tmp/tmp.4CmWCd8DEd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gLy04SbQNR + cat /tmp/tmp.4CmWCd8DEd + rm /tmp/tmp.gLy04SbQNR /tmp/tmp.4CmWCd8DEd + return 0 + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.KRWWUwR9VH ++ mktemp + local LAST_ERR=/tmp/tmp.Z0Lzor70iX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.41.154:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KRWWUwR9VH Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.36.41.154:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("9833929a-3f33-44b1-b6eb-0184b2c9a5bf") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match + cat /tmp/tmp.Z0Lzor70iX + rm /tmp/tmp.KRWWUwR9VH /tmp/tmp.Z0Lzor70iX + return 0 + desc 'run disaster recovery script for replset: rs0' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs0 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.36.44.28 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.36.44.28 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3Yd1tlVbSe +++ mktemp ++ local LAST_ERR=/tmp/tmp.gRUzrEyZu1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3Yd1tlVbSe ++ cat /tmp/tmp.gRUzrEyZu1 ++ rm /tmp/tmp.3Yd1tlVbSe /tmp/tmp.gRUzrEyZu1 ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.UWp3IkhKmk ++ mktemp + local LAST_ERR=/tmp/tmp.zORk0Ul5hI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UWp3IkhKmk + cat /tmp/tmp.zORk0Ul5hI + rm /tmp/tmp.UWp3IkhKmk /tmp/tmp.zORk0Ul5hI + return 0 + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.44.28:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.5h97RWTkBd ++ mktemp + local LAST_ERR=/tmp/tmp.UVabV8R47g + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.44.28:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5h97RWTkBd Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.36.44.28:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("a70016b7-f930-4d05-a975-577412f89263") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match + cat /tmp/tmp.UVabV8R47g + rm /tmp/tmp.5h97RWTkBd /tmp/tmp.UVabV8R47g + return 0 + desc 'run disaster recovery script for replset: rs1' + set +o xtrace ----------------------------------------------------------------------------------- run disaster recovery script for replset: rs1 ----------------------------------------------------------------------------------- + run_script_mongos /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js clusterAdmin:clusterAdmin123456@10.36.38.32 mongodb :27017 + local script=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + local uri=clusterAdmin:clusterAdmin123456@10.36.38.32 + local driver=mongodb + local suffix=:27017 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AeLJzyfmGb +++ mktemp ++ local LAST_ERR=/tmp/tmp.ScFKcXX21X ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AeLJzyfmGb ++ cat /tmp/tmp.ScFKcXX21X ++ rm /tmp/tmp.AeLJzyfmGb /tmp/tmp.ScFKcXX21X ++ return 0 + local client_container=psmdb-client-6c585f8dbd-hs4lz + local mongo_flag= ++ basename /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js + name=disaster_recovery.js + kubectl_bin cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp ++ mktemp + local LAST_OUT=/tmp/tmp.MKpgIEec1D ++ mktemp + local LAST_ERR=/tmp/tmp.KOL4Ku7Rk1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl cp /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/disaster_recovery.js cross-site-sharded-2508/psmdb-client-6c585f8dbd-hs4lz:/tmp + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MKpgIEec1D + cat /tmp/tmp.KOL4Ku7Rk1 + rm /tmp/tmp.MKpgIEec1D /tmp/tmp.KOL4Ku7Rk1 + return 0 + kubectl_bin exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.38.32:27017/admin /tmp/disaster_recovery.js' ++ mktemp + local LAST_OUT=/tmp/tmp.jvAt8UQ9QU ++ mktemp + local LAST_ERR=/tmp/tmp.Q1i7EJIMG7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-hs4lz -- bash -c 'mongo mongodb://clusterAdmin:clusterAdmin123456@10.36.38.32:27017/admin /tmp/disaster_recovery.js' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jvAt8UQ9QU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://10.36.38.32:27017/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("b669f058-ca79-4837-9d12-e6e856f1f440") } Percona Server for MongoDB server version: v7.0.14-8 WARNING: shell and server versions do not match + cat /tmp/tmp.Q1i7EJIMG7 + rm /tmp/tmp.jvAt8UQ9QU /tmp/tmp.Q1i7EJIMG7 + return 0 + desc 'make replica cluster managed' + set +o xtrace ----------------------------------------------------------------------------------- make replica cluster managed ----------------------------------------------------------------------------------- ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.3XpyBPs61p +++ mktemp ++ local LAST_ERR=/tmp/tmp.RShFHkx9cG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3XpyBPs61p ++ cat /tmp/tmp.RShFHkx9cG ++ rm /tmp/tmp.3XpyBPs61p /tmp/tmp.RShFHkx9cG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.uNwDiuRKY7 ++ mktemp + local LAST_ERR=/tmp/tmp.tl2rqqCPAA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7 --namespace=cross-site-sharded-replica-13364 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uNwDiuRKY7 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1647-9959fb75-26-cluster7" modified. + cat /tmp/tmp.tl2rqqCPAA + rm /tmp/tmp.uNwDiuRKY7 /tmp/tmp.tl2rqqCPAA + return 0 + kubectl_bin patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' ++ mktemp + local LAST_OUT=/tmp/tmp.6cJqIzWg4g ++ mktemp + local LAST_ERR=/tmp/tmp.jGyZsMhfGK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch psmdb cross-site-sharded-replica --type=merge --patch '{"spec":{"unmanaged": false}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6cJqIzWg4g perconaservermongodb.psmdb.percona.com/cross-site-sharded-replica patched + cat /tmp/tmp.jGyZsMhfGK + rm /tmp/tmp.6cJqIzWg4g /tmp/tmp.jGyZsMhfGK + return 0 + wait_for_running cross-site-sharded-replica-rs0 3 + local name=cross-site-sharded-replica-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-0 + local pod=cross-site-sharded-replica-rs0-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-rs0-1 + local pod=cross-site-sharded-replica-rs0-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cHLCB8ES9C +++ mktemp ++ local LAST_ERR=/tmp/tmp.hd7o9ftDah ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cHLCB8ES9C ++ cat /tmp/tmp.hd7o9ftDah ++ rm /tmp/tmp.cHLCB8ES9C /tmp/tmp.hd7o9ftDah ++ return 0 + [[ false == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-rs0-2 + local pod=cross-site-sharded-replica-rs0-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-rs0-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WosDnjFRDX +++ mktemp ++ local LAST_ERR=/tmp/tmp.k6jGzLnc5w ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WosDnjFRDX ++ cat /tmp/tmp.k6jGzLnc5w ++ rm /tmp/tmp.WosDnjFRDX /tmp/tmp.k6jGzLnc5w ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness. + wait_for_running cross-site-sharded-replica-cfg 3 + local name=cross-site-sharded-replica-cfg + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=cross-site-sharded-replica ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-0 + local pod=cross-site-sharded-replica-cfg-0 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod cross-site-sharded-replica-cfg-1 + local pod=cross-site-sharded-replica-cfg-1 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RHSNOjbiIU +++ mktemp ++ local LAST_ERR=/tmp/tmp.gsGDEe0Eco ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.RHSNOjbiIU ++ cat /tmp/tmp.gsGDEe0Eco ++ rm /tmp/tmp.RHSNOjbiIU /tmp/tmp.gsGDEe0Eco ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod cross-site-sharded-replica-cfg-2 + local pod=cross-site-sharded-replica-cfg-2 + set +o xtrace waiting for pod/cross-site-sharded-replica-cfg-2 to be ready.OK ++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GTZJiRf6t2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.IzQdJso78r ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GTZJiRf6t2 ++ cat /tmp/tmp.IzQdJso78r ++ rm /tmp/tmp.GTZJiRf6t2 /tmp/tmp.IzQdJso78r ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + desc 'check failover status' + set +o xtrace ----------------------------------------------------------------------------------- check failover status ----------------------------------------------------------------------------------- + compare_mongos_cmd find myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364 + local command=find + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364 + local postfix= + local suffix= + local database=myApp + local collection=test + run_mongos 'use myApp\n db.test.find()' myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364 mongodb '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' ++ local LAST_OUT=/tmp/tmp.y7b7Glghzf +++ mktemp ++ local LAST_ERR=/tmp/tmp.0SNyhw0ETk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.y7b7Glghzf ++ cat /tmp/tmp.0SNyhw0ETk ++ rm /tmp/tmp.y7b7Glghzf /tmp/tmp.0SNyhw0ETk ++ return 0 + local client_container=psmdb-client-6c585f8dbd-9x4mz + local mongo_flag= + kubectl_bin exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364.svc.cluster.local/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.8Lu8tdHuIh ++ mktemp + local LAST_ERR=/tmp/tmp.KTCcSLDVvs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-6c585f8dbd-9x4mz -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@cross-site-sharded-replica-mongos.cross-site-sharded-replica-13364.svc.cluster.local/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8Lu8tdHuIh + cat /tmp/tmp.KTCcSLDVvs + rm /tmp/tmp.8Lu8tdHuIh /tmp/tmp.KTCcSLDVvs + return 0 + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/e2e-tests/cross-site-sharded/compare/find.json /tmp/tmp.IhVzFBGGe8/find + desc 'Failover check finished successfully' + set +o xtrace ----------------------------------------------------------------------------------- Failover check finished successfully ----------------------------------------------------------------------------------- + wait_cluster_consistency cross-site-sharded-replica + local cluster_name=cross-site-sharded-replica + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cvjxneam9H +++ mktemp ++ local LAST_ERR=/tmp/tmp.FjgqyfVnqu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb cross-site-sharded-replica -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cvjxneam9H ++ cat /tmp/tmp.FjgqyfVnqu ++ rm /tmp/tmp.cvjxneam9H /tmp/tmp.FjgqyfVnqu ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo + destroy cross-site-sharded-2508 true + local namespace=cross-site-sharded-2508 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.H61C77TBkP ++ mktemp + local LAST_ERR=/tmp/tmp.QOACQnxsvC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.H61C77TBkP customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.QOACQnxsvC + rm /tmp/tmp.H61C77TBkP /tmp/tmp.QOACQnxsvC + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.KRkz478ML9 ++ mktemp + local LAST_ERR=/tmp/tmp.msucVSrTqo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KRkz478ML9 + cat /tmp/tmp.msucVSrTqo + rm /tmp/tmp.KRkz478ML9 /tmp/tmp.msucVSrTqo + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.u3ZOvx1MXE ++ mktemp + local LAST_ERR=/tmp/tmp.Dyek4Sa0U7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u3ZOvx1MXE + cat /tmp/tmp.Dyek4Sa0U7 + rm /tmp/tmp.u3ZOvx1MXE /tmp/tmp.Dyek4Sa0U7 + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.gJCFz5Unr0 ++ mktemp + local LAST_ERR=/tmp/tmp.CpfxQPdDim + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gJCFz5Unr0 + cat /tmp/tmp.CpfxQPdDim + rm /tmp/tmp.gJCFz5Unr0 /tmp/tmp.CpfxQPdDim + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.235vV8eLbp ++ mktemp + local LAST_ERR=/tmp/tmp.SExT8F082v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.235vV8eLbp role.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted serviceaccount "percona-server-mongodb-operator" deleted rolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.SExT8F082v + rm /tmp/tmp.235vV8eLbp /tmp/tmp.SExT8F082v + return 0 + destroy_cert_manager + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.so41TPSCzl ++ mktemp + local LAST_ERR=/tmp/tmp.O2iveollBf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.so41TPSCzl + cat /tmp/tmp.O2iveollBf Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.so41TPSCzl + cat /tmp/tmp.O2iveollBf Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.so41TPSCzl + cat /tmp/tmp.O2iveollBf Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.so41TPSCzl + cat /tmp/tmp.O2iveollBf Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.so41TPSCzl /tmp/tmp.O2iveollBf + return 1 + true + '[' -n '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.IhVzFBGGe8 + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-2508 + destroy cross-site-sharded-replica-13364 true + local namespace=cross-site-sharded-replica-13364 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false ']' + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.3wKPjN2Qi5 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.KYckqudyZv ++ mktemp + local LAST_ERR=/tmp/tmp.UjGer0JWIp + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.kNUyqRw0VG + local exit_status=0 + local timeout=4 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml --ignore-not-found --wait=false ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-2508 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KYckqudyZv + cat /tmp/tmp.UjGer0JWIp + rm /tmp/tmp.KYckqudyZv /tmp/tmp.UjGer0JWIp + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/crd.yaml + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbbackups" + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbbackups" + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.jThfrbxuLb ++ mktemp + local LAST_ERR=/tmp/tmp.cI50MrNkKE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jThfrbxuLb + cat /tmp/tmp.cI50MrNkKE + rm /tmp/tmp.jThfrbxuLb /tmp/tmp.cI50MrNkKE + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JWkTIEXsjp ++ mktemp + local LAST_ERR=/tmp/tmp.Txxmj6Sq3G + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JWkTIEXsjp + cat /tmp/tmp.Txxmj6Sq3G + rm /tmp/tmp.JWkTIEXsjp /tmp/tmp.Txxmj6Sq3G + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.yPGp6Xh8QQ ++ mktemp + local LAST_ERR=/tmp/tmp.pbhBz2hPAq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yPGp6Xh8QQ + cat /tmp/tmp.pbhBz2hPAq + rm /tmp/tmp.yPGp6Xh8QQ /tmp/tmp.pbhBz2hPAq + return 0 + local rbac_yaml=rbac.yaml + '[' -n '' ']' + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.JOlZ1pQeug ++ mktemp + local LAST_ERR=/tmp/tmp.wclFNpQtUG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1647/deploy/rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JOlZ1pQeug + cat /tmp/tmp.wclFNpQtUG + rm /tmp/tmp.JOlZ1pQeug /tmp/tmp.wclFNpQtUG + return 0 + destroy_cert_manager + kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.YRdvZCMs50 ++ mktemp + local LAST_ERR=/tmp/tmp.jYTMAFItyo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3wKPjN2Qi5 namespace "cross-site-sharded-2508" force deleted + cat /tmp/tmp.kNUyqRw0VG Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.3wKPjN2Qi5 /tmp/tmp.kNUyqRw0VG + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.YRdvZCMs50 + cat /tmp/tmp.jYTMAFItyo Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.YRdvZCMs50 + cat /tmp/tmp.jYTMAFItyo Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 4 + for i in '$(seq 0 2)' + set +e + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.YRdvZCMs50 + cat /tmp/tmp.jYTMAFItyo Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + sleep 8 + cat /tmp/tmp.YRdvZCMs50 + cat /tmp/tmp.jYTMAFItyo Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found + rm /tmp/tmp.YRdvZCMs50 /tmp/tmp.jYTMAFItyo + return 1 + true + '[' -n '' ']' + '[' -n '' ']' + rm -rf /tmp/tmp.IhVzFBGGe8 + kubectl_bin delete --grace-period=0 --force=true namespace cross-site-sharded-replica-13364 ++ mktemp + local LAST_OUT=/tmp/tmp.qSXu3e2NX4 ++ mktemp + local LAST_ERR=/tmp/tmp.g9C16CO7T1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace cross-site-sharded-replica-13364 + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------