Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/logs/demand-backup-snapshot.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 ++ deploy_volume_snapshot_class ++ local snapshot_class_file ++ local k8s_env +++ detect_k8s_env +++ local provider=unknown +++ kubectl_bin api-resources +++ grep -q openshift.io ++++ mktemp +++ local LAST_OUT=/tmp/tmp.49KOrwult9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.my55bYp6wm +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl api-resources +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.49KOrwult9 +++ cat /tmp/tmp.my55bYp6wm +++ rm /tmp/tmp.49KOrwult9 /tmp/tmp.my55bYp6wm +++ return 0 +++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}' +++ grep -q eks.amazonaws.com +++ kubectl_bin get nodes -o 'jsonpath={.items[0].metadata.labels}' +++ grep -q cloud.google.com/gke +++ provider=gke +++ echo gke ++ k8s_env=gke ++ case "$k8s_env" in ++ snapshot_class_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/gke-volume-snapshot-class.yaml ++ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/gke-volume-snapshot-class.yaml +++ mktemp ++ local LAST_OUT=/tmp/tmp.8kawS3aLDB +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ot2vMSaYdD ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/gke-volume-snapshot-class.yaml ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8kawS3aLDB volumesnapshotclass.snapshot.storage.k8s.io/gke-snapshot-class unchanged ++ cat /tmp/tmp.Ot2vMSaYdD ++ rm /tmp/tmp.8kawS3aLDB /tmp/tmp.Ot2vMSaYdD ++ return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/gke-volume-snapshot-class.yaml + VOLUME_SNAPSHOT_CLASS=gke-snapshot-class + create_infra demand-backup-snapshot-13296 + local ns=demand-backup-snapshot-13296 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.88OKarBFD1 ++ mktemp + local LAST_ERR=/tmp/tmp.qbAn0WJs9Y + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.88OKarBFD1 customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.qbAn0WJs9Y + rm /tmp/tmp.88OKarBFD1 /tmp/tmp.qbAn0WJs9Y + return 0 ++ grep -v '\-\-\-' ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-snapshot-19001 backup-snapshot --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodbbackup.psmdb.percona.com/backup-snapshot patched + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.JXXYxrAzza ++ mktemp + local LAST_ERR=/tmp/tmp.pVAhvyeDRq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JXXYxrAzza + cat /tmp/tmp.pVAhvyeDRq + rm /tmp/tmp.JXXYxrAzza /tmp/tmp.pVAhvyeDRq + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.ntw2gcFJ1S ++ mktemp + local LAST_ERR=/tmp/tmp.ppJRryXf67 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ntw2gcFJ1S + cat /tmp/tmp.ppJRryXf67 + rm /tmp/tmp.ntw2gcFJ1S /tmp/tmp.ppJRryXf67 + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch perconaservermongodbs.psmdb.percona.com -n demand-backup-snapshot-19001 some-name --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodb.psmdb.percona.com/some-name patched + kubectl patch perconaservermongodbs.psmdb.percona.com -n demand-backup-snapshot-19001 some-name-new --type=merge -p '{"metadata":{"finalizers":[]}}' perconaservermongodb.psmdb.percona.com/some-name-new patched + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.foYH79FvDj ++ mktemp + local LAST_ERR=/tmp/tmp.MFBIQazw2O + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.foYH79FvDj customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met + cat /tmp/tmp.MFBIQazw2O + rm /tmp/tmp.foYH79FvDj /tmp/tmp.MFBIQazw2O + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.oTnTseOIvP ++ mktemp + local LAST_ERR=/tmp/tmp.mP4mlrl2NV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oTnTseOIvP clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.mP4mlrl2NV + rm /tmp/tmp.oTnTseOIvP /tmp/tmp.mP4mlrl2NV + return 0 + check_crd_for_deletion PR-2271-c31e8c20 + local git_tag=PR-2271-c31e8c20 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2271-c31e8c20/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed s/---//g ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fq2EBRuqVM +++ mktemp ++ local LAST_ERR=/tmp/tmp.xoCk2RAalA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fq2EBRuqVM ++ cat /tmp/tmp.xoCk2RAalA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fq2EBRuqVM ++ cat /tmp/tmp.xoCk2RAalA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.fq2EBRuqVM ++ cat /tmp/tmp.xoCk2RAalA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.fq2EBRuqVM ++ cat /tmp/tmp.xoCk2RAalA Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.fq2EBRuqVM /tmp/tmp.xoCk2RAalA ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + xargs kubectl delete ns + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.6o8sOZ4Oeu + local LAST_OUT=/tmp/tmp.gCgT3t4vwA ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.STdcvqsk5F + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.sAk5DGgET4 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gCgT3t4vwA + cat /tmp/tmp.sAk5DGgET4 + rm /tmp/tmp.gCgT3t4vwA /tmp/tmp.sAk5DGgET4 + return 0 namespace "demand-backup-snapshot-19001" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6o8sOZ4Oeu namespace "psmdb-operator" deleted + cat /tmp/tmp.STdcvqsk5F + rm /tmp/tmp.6o8sOZ4Oeu /tmp/tmp.STdcvqsk5F + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mwEjfLiOgK ++ mktemp + local LAST_ERR=/tmp/tmp.qVlaaVpgvv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mwEjfLiOgK + cat /tmp/tmp.qVlaaVpgvv + rm /tmp/tmp.mwEjfLiOgK /tmp/tmp.qVlaaVpgvv + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.uBWsnmtRlp ++ mktemp + local LAST_ERR=/tmp/tmp.ZPzZPtMrqn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uBWsnmtRlp namespace/psmdb-operator created + cat /tmp/tmp.ZPzZPtMrqn + rm /tmp/tmp.uBWsnmtRlp /tmp/tmp.ZPzZPtMrqn + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.OrK7KVlyCP +++ mktemp ++ local LAST_ERR=/tmp/tmp.TxIRcHF5NI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OrK7KVlyCP ++ cat /tmp/tmp.TxIRcHF5NI ++ rm /tmp/tmp.OrK7KVlyCP /tmp/tmp.TxIRcHF5NI ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.UtoCiXSYhs ++ mktemp + local LAST_ERR=/tmp/tmp.fpUhfKTvol + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.UtoCiXSYhs Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3" modified. + cat /tmp/tmp.fpUhfKTvol + rm /tmp/tmp.UtoCiXSYhs /tmp/tmp.fpUhfKTvol + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.zzoo9DubIf ++ mktemp + local LAST_ERR=/tmp/tmp.LZ79GfzFBA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zzoo9DubIf customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.LZ79GfzFBA + rm /tmp/tmp.zzoo9DubIf /tmp/tmp.LZ79GfzFBA + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tIPBuwfGIe ++ mktemp + local LAST_ERR=/tmp/tmp.X0t86cuQec + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tIPBuwfGIe clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.X0t86cuQec + rm /tmp/tmp.tIPBuwfGIe /tmp/tmp.X0t86cuQec + return 0 + yq eval ' (.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kVF2it2ktJ ++ mktemp + local LAST_ERR=/tmp/tmp.AKX09W0E1v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kVF2it2ktJ deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.AKX09W0E1v + rm /tmp/tmp.kVF2it2ktJ /tmp/tmp.AKX09W0E1v + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.bi1vBnqHmk +++ mktemp ++ local LAST_ERR=/tmp/tmp.Zoa34d0pHm ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bi1vBnqHmk ++ cat /tmp/tmp.Zoa34d0pHm ++ rm /tmp/tmp.bi1vBnqHmk /tmp/tmp.Zoa34d0pHm ++ return 0 + wait_operator_pod percona-server-mongodb-operator-59f5f7f8b4-r67mq + local pod=percona-server-mongodb-operator-59f5f7f8b4-r67mq + set +o xtrace waiting for pod/percona-server-mongodb-operator-59f5f7f8b4-r67mq to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.0bzVfDBQPb +++ mktemp ++ local LAST_ERR=/tmp/tmp.nQK7rGXpMU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0bzVfDBQPb ++ cat /tmp/tmp.nQK7rGXpMU ++ rm /tmp/tmp.0bzVfDBQPb /tmp/tmp.nQK7rGXpMU ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-59f5f7f8b4-r67mq ++ mktemp + local LAST_OUT=/tmp/tmp.5ZqoWzHknl ++ mktemp + local LAST_ERR=/tmp/tmp.rjMkhNaeeD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-59f5f7f8b4-r67mq + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5ZqoWzHknl + cat /tmp/tmp.rjMkhNaeeD + rm /tmp/tmp.5ZqoWzHknl /tmp/tmp.rjMkhNaeeD + return 0 2026-03-26T09:13:32.082Z INFO setup Manager starting up {"gitCommit": "c31e8c20f2a920146c4a2410dc38a83148f57a2b", "gitBranch": "PR-2271-c31e8c20", "buildTime": "", "goVersion": "go1.25.8", "os": "linux", "arch": "amd64"} + create_namespace demand-backup-snapshot-13296 + local namespace=demand-backup-snapshot-13296 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-snapshot-13296' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-snapshot-13296 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-snapshot-13296 --ignore-not-found + awk '{print$1}' + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.bl2Ddvz4hW ++ mktemp + local LAST_OUT=/tmp/tmp.Qd70tSLIhF ++ mktemp + local LAST_ERR=/tmp/tmp.7rfxOhZEmE + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.TcSGpcoLmf + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace demand-backup-snapshot-13296 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.bl2Ddvz4hW + cat /tmp/tmp.7rfxOhZEmE + rm /tmp/tmp.bl2Ddvz4hW /tmp/tmp.7rfxOhZEmE + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Qd70tSLIhF + cat /tmp/tmp.TcSGpcoLmf + rm /tmp/tmp.Qd70tSLIhF /tmp/tmp.TcSGpcoLmf + return 0 + kubectl_bin wait --for=delete namespace demand-backup-snapshot-13296 ++ mktemp + local LAST_OUT=/tmp/tmp.TkMwsJyZE6 ++ mktemp + local LAST_ERR=/tmp/tmp.3Itn1gQ0ev + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace demand-backup-snapshot-13296 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TkMwsJyZE6 + cat /tmp/tmp.3Itn1gQ0ev + rm /tmp/tmp.TkMwsJyZE6 /tmp/tmp.3Itn1gQ0ev + return 0 + desc 'create namespace demand-backup-snapshot-13296' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-snapshot-13296 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-snapshot-13296 ++ mktemp + local LAST_OUT=/tmp/tmp.3yHMFYfagZ ++ mktemp + local LAST_ERR=/tmp/tmp.a7Bzukzj1J + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace demand-backup-snapshot-13296 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.3yHMFYfagZ namespace/demand-backup-snapshot-13296 created + cat /tmp/tmp.a7Bzukzj1J + rm /tmp/tmp.3yHMFYfagZ /tmp/tmp.a7Bzukzj1J + return 0 + set_kube_ctx demand-backup-snapshot-13296 + local namespace=demand-backup-snapshot-13296 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.aq2FiPBvst +++ mktemp ++ local LAST_ERR=/tmp/tmp.48v9p7Hwcc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.aq2FiPBvst ++ cat /tmp/tmp.48v9p7Hwcc ++ rm /tmp/tmp.aq2FiPBvst /tmp/tmp.48v9p7Hwcc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=demand-backup-snapshot-13296 ++ mktemp + local LAST_OUT=/tmp/tmp.daHSvmhQxj ++ mktemp + local LAST_ERR=/tmp/tmp.0WEDeEUKcT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3 --namespace=demand-backup-snapshot-13296 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.daHSvmhQxj Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2271-c31e8c20-4-cluster3" modified. + cat /tmp/tmp.0WEDeEUKcT + rm /tmp/tmp.daHSvmhQxj /tmp/tmp.0WEDeEUKcT + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Thu Mar 26 09:14:08 2026 NAMESPACE: demand-backup-snapshot-13296 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-snapshot-13296.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-snapshot-13296 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-snapshot-13296 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-snapshot-13296 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-snapshot-13296 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pz0cQytjYU +++ mktemp ++ local LAST_ERR=/tmp/tmp.rc0YPtBK6E ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Pz0cQytjYU ++ cat /tmp/tmp.rc0YPtBK6E ++ rm /tmp/tmp.Pz0cQytjYU /tmp/tmp.rc0YPtBK6E ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-vx6jr + wait_pod minio-service-6d5f646cdc-vx6jr + local pod=minio-service-6d5f646cdc-vx6jr + set +o xtrace waiting for pod/minio-service-6d5f646cdc-vx6jr to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-snapshot-13296.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.pWgmSn1hfB ++ mktemp + local LAST_ERR=/tmp/tmp.8t2KPDm6hu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pWgmSn1hfB make_bucket: operator-testing pod "aws-cli" deleted from demand-backup-snapshot-13296 namespace + cat /tmp/tmp.8t2KPDm6hu All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-snapshot-13296 + rm /tmp/tmp.pWgmSn1hfB /tmp/tmp.8t2KPDm6hu + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.Lp5uRnC9Vb ++ mktemp + local LAST_ERR=/tmp/tmp.ZxzAc0HBLl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Lp5uRnC9Vb secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created secret/gcp-cs-sa-key-secret created + cat /tmp/tmp.ZxzAc0HBLl + rm /tmp/tmp.Lp5uRnC9Vb /tmp/tmp.ZxzAc0HBLl + return 0 + cluster=some-name + desc 'Testing snapshot backup/restore on unencrypted cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing snapshot backup/restore on unencrypted cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.00dBnsmYcS ++ mktemp + local LAST_ERR=/tmp/tmp.ESbZJ8dltN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.00dBnsmYcS secret/some-users created + cat /tmp/tmp.ESbZJ8dltN + rm /tmp/tmp.00dBnsmYcS /tmp/tmp.ESbZJ8dltN + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name.yml ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-snapshot-13296/g + local LAST_OUT=/tmp/tmp.Lpnb5oHVeD + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.4VrVkZywk3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Lpnb5oHVeD perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.4VrVkZywk3 + rm /tmp/tmp.Lpnb5oHVeD /tmp/tmp.4VrVkZywk3 + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client_with_tls.yml ++ mktemp + local LAST_OUT=/tmp/tmp.I9LmwJbF5w ++ mktemp + local LAST_ERR=/tmp/tmp.Z0RH6N1ThZ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/conf/client_with_tls.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.I9LmwJbF5w deployment.apps/psmdb-client created + cat /tmp/tmp.Z0RH6N1ThZ + rm /tmp/tmp.I9LmwJbF5w /tmp/tmp.Z0RH6N1ThZ + return 0 + echo 'check if all pods started' check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YtiI6oDiqG +++ mktemp ++ local LAST_ERR=/tmp/tmp.IwBO0Kfy4z ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YtiI6oDiqG ++ cat /tmp/tmp.IwBO0Kfy4z ++ rm /tmp/tmp.YtiI6oDiqG /tmp/tmp.IwBO0Kfy4z ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nq2DX29bxk +++ mktemp ++ local LAST_ERR=/tmp/tmp.SXAKVwYuWi ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nq2DX29bxk ++ cat /tmp/tmp.SXAKVwYuWi ++ rm /tmp/tmp.nq2DX29bxk /tmp/tmp.SXAKVwYuWi ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZtQULqDlz4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wZWBIkKjPI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZtQULqDlz4 ++ cat /tmp/tmp.wZWBIkKjPI ++ rm /tmp/tmp.ZtQULqDlz4 /tmp/tmp.wZWBIkKjPI ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EVbPGAse4h +++ mktemp ++ local LAST_ERR=/tmp/tmp.EXlrfHDo1M ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EVbPGAse4h ++ cat /tmp/tmp.EXlrfHDo1M ++ rm /tmp/tmp.EVbPGAse4h /tmp/tmp.EXlrfHDo1M ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + echo 'Writing test data' Writing test data + run_mongo_tls 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AR01xkSYcW +++ mktemp ++ local LAST_ERR=/tmp/tmp.cGRNCQePZb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.AR01xkSYcW ++ cat /tmp/tmp.cGRNCQePZb ++ rm /tmp/tmp.AR01xkSYcW /tmp/tmp.cGRNCQePZb ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.T4CNHWblVA ++ mktemp + local LAST_ERR=/tmp/tmp.KZzFUC2gSO + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.T4CNHWblVA Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:17:30.626Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("bcf3c136-2b92-4ddd-b763-29dde5b7e44b") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.KZzFUC2gSO + rm /tmp/tmp.T4CNHWblVA /tmp/tmp.KZzFUC2gSO + return 0 + sleep 1 + run_mongo_tls 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hA1jLTmHUf +++ mktemp ++ local LAST_ERR=/tmp/tmp.BTjr1F8OLd ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hA1jLTmHUf ++ cat /tmp/tmp.BTjr1F8OLd ++ rm /tmp/tmp.hA1jLTmHUf /tmp/tmp.BTjr1F8OLd ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.jJD6qmLyOx ++ mktemp + local LAST_ERR=/tmp/tmp.7XYqUfbkiQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jJD6qmLyOx Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:17:34.327Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("40a6ac19-fa18-40a3-8ae4-0c2646351e07") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.7XYqUfbkiQ + rm /tmp/tmp.jJD6qmLyOx /tmp/tmp.7XYqUfbkiQ + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:17:39+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gZCLAvjLOD +++ mktemp ++ local LAST_ERR=/tmp/tmp.4xVp2LWqGK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gZCLAvjLOD ++ cat /tmp/tmp.4xVp2LWqGK ++ rm /tmp/tmp.gZCLAvjLOD /tmp/tmp.4xVp2LWqGK ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.52MmrLtQU6 ++ mktemp + local LAST_ERR=/tmp/tmp.q4eyVtDuUl + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.52MmrLtQU6 + cat /tmp/tmp.q4eyVtDuUl + rm /tmp/tmp.52MmrLtQU6 /tmp/tmp.q4eyVtDuUl + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:17:42+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' +++ mktemp ++ local LAST_OUT=/tmp/tmp.37wMOKHWqf +++ mktemp ++ local LAST_ERR=/tmp/tmp.adge7U6n52 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.37wMOKHWqf ++ cat /tmp/tmp.adge7U6n52 ++ rm /tmp/tmp.37wMOKHWqf /tmp/tmp.adge7U6n52 ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.L3woxC0Ihn ++ mktemp + local LAST_ERR=/tmp/tmp.uCA1uGi739 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L3woxC0Ihn + cat /tmp/tmp.uCA1uGi739 + rm /tmp/tmp.L3woxC0Ihn /tmp/tmp.uCA1uGi739 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:17:44+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VJK4yKVJwQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZVP8wi7eIx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VJK4yKVJwQ ++ cat /tmp/tmp.ZVP8wi7eIx ++ rm /tmp/tmp.VJK4yKVJwQ /tmp/tmp.ZVP8wi7eIx ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.sbObcvogyH ++ mktemp + local LAST_ERR=/tmp/tmp.gV72ld2nBL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sbObcvogyH + cat /tmp/tmp.gV72ld2nBL + rm /tmp/tmp.sbObcvogyH /tmp/tmp.gV72ld2nBL + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + echo 'Running snapshot backup (unencrypted)' Running snapshot backup (unencrypted) + backup_name=backup-snapshot + run_snapshot_backup backup-snapshot + local backup_name=backup-snapshot + log 'running snapshot backup backup-snapshot' + set +o xtrace [2026-03-26T09:17:47+0000] running snapshot backup backup-snapshot + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-snapshot" | .spec.volumeSnapshotClass = "gke-snapshot-class"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/backup.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hO3WjE6Dtl ++ mktemp + local LAST_ERR=/tmp/tmp.Z0VacKa545 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hO3WjE6Dtl perconaservermongodbbackup.psmdb.percona.com/backup-snapshot created + cat /tmp/tmp.Z0VacKa545 + rm /tmp/tmp.hO3WjE6Dtl /tmp/tmp.Z0VacKa545 + return 0 + wait_backup backup-snapshot + local backup_name=backup-snapshot + local target_state=ready + set +o xtrace waiting for backup-snapshot to reach ready state.......................OK + echo 'Drop collection and restore from snapshot (unencrypted)' Drop collection and restore from snapshot (unencrypted) + run_mongo_tls 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X4cpmwmBTi +++ mktemp ++ local LAST_ERR=/tmp/tmp.0NWN3hEkyg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.X4cpmwmBTi ++ cat /tmp/tmp.0NWN3hEkyg ++ rm /tmp/tmp.X4cpmwmBTi /tmp/tmp.0NWN3hEkyg ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.qGfl7dEfs9 ++ mktemp + local LAST_ERR=/tmp/tmp.TjpKYBBgQ0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qGfl7dEfs9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:18:36.266Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("0dc55d87-f712-414b-b404-04f08a0adf30") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.TjpKYBBgQ0 + rm /tmp/tmp.qGfl7dEfs9 /tmp/tmp.TjpKYBBgQ0 + return 0 + run_restore backup-snapshot + local backup_name=backup-snapshot + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/restore.yml + log 'running restore restore-backup-snapshot' + set +o xtrace [2026-03-26T09:18:36+0000] running restore restore-backup-snapshot + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-snapshot/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-snapshot/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ZY6GKu2BIe ++ mktemp + local LAST_ERR=/tmp/tmp.3z85wkgChg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZY6GKu2BIe perconaservermongodbrestore.psmdb.percona.com/restore-backup-snapshot created + cat /tmp/tmp.3z85wkgChg + rm /tmp/tmp.ZY6GKu2BIe /tmp/tmp.3z85wkgChg + return 0 + run_snapshot_recovery_check backup-snapshot + local backup_name=backup-snapshot + local target_cluster=some-name + wait_restore backup-snapshot some-name ready 0 3000 + local backup_name=backup-snapshot + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-snapshot object to be created.OK Waiting psmdb-restore/restore-backup-snapshot to reach state "ready" .......OK after 6 minutes + [[ 0 -eq 1 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.metadata.annotations.percona\.com/resync-pbm}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Pa4JOSrxfT +++ mktemp ++ local LAST_ERR=/tmp/tmp.zJTKm1epa8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.metadata.annotations.percona\.com/resync-pbm}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Pa4JOSrxfT ++ cat /tmp/tmp.zJTKm1epa8 ++ rm /tmp/tmp.Pa4JOSrxfT /tmp/tmp.zJTKm1epa8 ++ return 0 + '[' true '!=' true ']' + log 'Operator triggered PBM resync: OK' + set +o xtrace [2026-03-26T09:25:35+0000] Operator triggered PBM resync: OK + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Psqel29M7j +++ mktemp ++ local LAST_ERR=/tmp/tmp.ybZVRdBliv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Psqel29M7j ++ cat /tmp/tmp.ybZVRdBliv ++ rm /tmp/tmp.Psqel29M7j /tmp/tmp.ybZVRdBliv ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KN273VljzO +++ mktemp ++ local LAST_ERR=/tmp/tmp.jdExPJ69er ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KN273VljzO ++ cat /tmp/tmp.jdExPJ69er ++ rm /tmp/tmp.KN273VljzO /tmp/tmp.jdExPJ69er ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.G6O3da556q +++ mktemp ++ local LAST_ERR=/tmp/tmp.Yj5CS8nWgy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.G6O3da556q ++ cat /tmp/tmp.Yj5CS8nWgy ++ rm /tmp/tmp.G6O3da556q /tmp/tmp.Yj5CS8nWgy ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tM8ApqJ8Xj +++ mktemp ++ local LAST_ERR=/tmp/tmp.yXwV3xXAAu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.tM8ApqJ8Xj ++ cat /tmp/tmp.yXwV3xXAAu ++ rm /tmp/tmp.tM8ApqJ8Xj /tmp/tmp.yXwV3xXAAu ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JU5Esi2PqF +++ mktemp ++ local LAST_ERR=/tmp/tmp.6PaPZwGtRT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JU5Esi2PqF ++ cat /tmp/tmp.6PaPZwGtRT ++ rm /tmp/tmp.JU5Esi2PqF /tmp/tmp.6PaPZwGtRT ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 5 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zGU4QMWXoO +++ mktemp ++ local LAST_ERR=/tmp/tmp.rRcrkZpytV ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zGU4QMWXoO ++ cat /tmp/tmp.rRcrkZpytV ++ rm /tmp/tmp.zGU4QMWXoO /tmp/tmp.rRcrkZpytV ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 6 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.D9ckG9jQfN +++ mktemp ++ local LAST_ERR=/tmp/tmp.zAjCcnXJuc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.D9ckG9jQfN ++ cat /tmp/tmp.zAjCcnXJuc ++ rm /tmp/tmp.D9ckG9jQfN /tmp/tmp.zAjCcnXJuc ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 7 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kjojglwDjN +++ mktemp ++ local LAST_ERR=/tmp/tmp.JtfAsShA9s ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kjojglwDjN ++ cat /tmp/tmp.JtfAsShA9s ++ rm /tmp/tmp.kjojglwDjN /tmp/tmp.JtfAsShA9s ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 8 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Biitsyf7PY +++ mktemp ++ local LAST_ERR=/tmp/tmp.2C6kg5wtWo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Biitsyf7PY ++ cat /tmp/tmp.2C6kg5wtWo ++ rm /tmp/tmp.Biitsyf7PY /tmp/tmp.2C6kg5wtWo ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 9 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.UgSkIFUZH4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9UqrBG4G9X ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.UgSkIFUZH4 ++ cat /tmp/tmp.9UqrBG4G9X ++ rm /tmp/tmp.UgSkIFUZH4 /tmp/tmp.9UqrBG4G9X ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:27:24+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1eYKz8rTsm +++ mktemp ++ local LAST_ERR=/tmp/tmp.D9GphoO51W ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.1eYKz8rTsm ++ cat /tmp/tmp.D9GphoO51W ++ rm /tmp/tmp.1eYKz8rTsm /tmp/tmp.D9GphoO51W ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.S6bWCphnyr ++ mktemp + local LAST_ERR=/tmp/tmp.L9FpK3TkCJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S6bWCphnyr + cat /tmp/tmp.L9FpK3TkCJ + rm /tmp/tmp.S6bWCphnyr /tmp/tmp.L9FpK3TkCJ + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:27:26+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QynXfpkxmz +++ mktemp ++ local LAST_ERR=/tmp/tmp.L3upuhaDtt ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QynXfpkxmz ++ cat /tmp/tmp.L3upuhaDtt ++ rm /tmp/tmp.QynXfpkxmz /tmp/tmp.L3upuhaDtt ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.kXMoiAI7Cx ++ mktemp + local LAST_ERR=/tmp/tmp.LfAAqF3qOT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kXMoiAI7Cx + cat /tmp/tmp.LfAAqF3qOT + rm /tmp/tmp.kXMoiAI7Cx /tmp/tmp.LfAAqF3qOT + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:27:28+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.69XxMF2KVz +++ mktemp ++ local LAST_ERR=/tmp/tmp.Tdzaj72RNX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.69XxMF2KVz ++ cat /tmp/tmp.Tdzaj72RNX ++ rm /tmp/tmp.69XxMF2KVz /tmp/tmp.Tdzaj72RNX ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.U9sw0Irk8k ++ mktemp + local LAST_ERR=/tmp/tmp.Mdi6dvQ3I4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U9sw0Irk8k + cat /tmp/tmp.Mdi6dvQ3I4 + rm /tmp/tmp.U9sw0Irk8k /tmp/tmp.Mdi6dvQ3I4 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + log 'Data restored: OK' + set +o xtrace [2026-03-26T09:27:31+0000] Data restored: OK + desc 'Testing cross-cluster snapshot restore to a new cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing cross-cluster snapshot restore to a new cluster ----------------------------------------------------------------------------------- + new_cluster=some-name-new + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-new.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-new.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-new.yml + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-snapshot-13296/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_OUT=/tmp/tmp.iguJ3rNpQG ++ mktemp + local LAST_ERR=/tmp/tmp.Wob8Jff1Uq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iguJ3rNpQG perconaservermongodb.psmdb.percona.com/some-name-new created + cat /tmp/tmp.Wob8Jff1Uq + rm /tmp/tmp.iguJ3rNpQG /tmp/tmp.Wob8Jff1Uq + return 0 + echo 'check if all pods started on new cluster' check if all pods started on new cluster + wait_for_running some-name-new-rs0 3 + local name=some-name-new-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name-new ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-new-rs0-0 + local pod=some-name-new-rs0-0 + set +o xtrace waiting for pod/some-name-new-rs0-0 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-new-rs0-1 + local pod=some-name-new-rs0-1 + set +o xtrace waiting for pod/some-name-new-rs0-1 to be ready.....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uo3kJmKwdX +++ mktemp ++ local LAST_ERR=/tmp/tmp.QgcyHubVAE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uo3kJmKwdX ++ cat /tmp/tmp.QgcyHubVAE ++ rm /tmp/tmp.uo3kJmKwdX /tmp/tmp.QgcyHubVAE ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-new-rs0-2 + local pod=some-name-new-rs0-2 + set +o xtrace waiting for pod/some-name-new-rs0-2 to be ready..................OK ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4RPNyPC7JW +++ mktemp ++ local LAST_ERR=/tmp/tmp.WnBKyzHc50 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4RPNyPC7JW ++ cat /tmp/tmp.WnBKyzHc50 ++ rm /tmp/tmp.4RPNyPC7JW /tmp/tmp.WnBKyzHc50 ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8CSEMXoRNA +++ mktemp ++ local LAST_ERR=/tmp/tmp.LAfUuthyai ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.8CSEMXoRNA ++ cat /tmp/tmp.LAfUuthyai ++ rm /tmp/tmp.8CSEMXoRNA /tmp/tmp.LAfUuthyai ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness..... + wait_cluster_consistency some-name-new + local cluster_name=some-name-new + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5bM5nKVx9u +++ mktemp ++ local LAST_ERR=/tmp/tmp.6zDtJ84pmw ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5bM5nKVx9u ++ cat /tmp/tmp.6zDtJ84pmw ++ rm /tmp/tmp.5bM5nKVx9u /tmp/tmp.6zDtJ84pmw ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name-new + local cluster=some-name-new + set +o xtrace waiting for PBM operation to finish.OK + echo 'Restoring snapshot to new cluster using backupSource' Restoring snapshot to new cluster using backupSource + run_snapshot_restore_backupsource backup-snapshot some-name-new + local backup_name=backup-snapshot + local target_cluster=some-name-new + local snapshots ++ kubectl_bin get psmdb-backup backup-snapshot -o 'jsonpath={.status.snapshots}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uuS74DdPgU +++ mktemp ++ local LAST_ERR=/tmp/tmp.2MYwtrqgeC ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-snapshot -o 'jsonpath={.status.snapshots}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.uuS74DdPgU ++ cat /tmp/tmp.2MYwtrqgeC ++ rm /tmp/tmp.uuS74DdPgU /tmp/tmp.2MYwtrqgeC ++ return 0 + snapshots='[{"replsetName":"rs0","snapshotName":"backup-snapshot-rs0"}]' + log 'running cross-cluster snapshot restore to some-name-new from backup backup-snapshot' + set +o xtrace [2026-03-26T09:30:39+0000] running cross-cluster snapshot restore to some-name-new from backup backup-snapshot + backup_name=backup-snapshot + target_cluster=some-name-new + snapshots='[{"replsetName":"rs0","snapshotName":"backup-snapshot-rs0"}]' + yq eval ' .metadata.name = "restore-backupsource-" + strenv(backup_name) | .spec.clusterName = strenv(target_cluster) | .spec.backupSource.snapshots = env(snapshots) ' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/restore-backupsource.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ozZboCMPoL ++ mktemp + local LAST_ERR=/tmp/tmp.88Ks2e4dqp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ozZboCMPoL perconaservermongodbrestore.psmdb.percona.com/restore-backupsource-backup-snapshot created + cat /tmp/tmp.88Ks2e4dqp + rm /tmp/tmp.ozZboCMPoL /tmp/tmp.88Ks2e4dqp + return 0 + run_snapshot_recovery_check backupsource-backup-snapshot some-name-new + local backup_name=backupsource-backup-snapshot + local target_cluster=some-name-new + wait_restore backupsource-backup-snapshot some-name-new ready 0 3000 + local backup_name=backupsource-backup-snapshot + local cluster_name=some-name-new + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backupsource-backup-snapshot object to be created.OK Waiting psmdb-restore/restore-backupsource-backup-snapshot to reach state "ready" .......OK after 6 minutes + [[ 0 -eq 1 ]] ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.metadata.annotations.percona\.com/resync-pbm}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3ogVHvaQ0U +++ mktemp ++ local LAST_ERR=/tmp/tmp.hELYjkx9q4 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.metadata.annotations.percona\.com/resync-pbm}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3ogVHvaQ0U ++ cat /tmp/tmp.hELYjkx9q4 ++ rm /tmp/tmp.3ogVHvaQ0U /tmp/tmp.hELYjkx9q4 ++ return 0 + '[' true '!=' true ']' + log 'Operator triggered PBM resync: OK' + set +o xtrace [2026-03-26T09:37:05+0000] Operator triggered PBM resync: OK + wait_cluster_consistency some-name-new + local cluster_name=some-name-new + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w1ZQfg9Wn6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aNjXrrl2im ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.w1ZQfg9Wn6 ++ cat /tmp/tmp.aNjXrrl2im ++ rm /tmp/tmp.w1ZQfg9Wn6 /tmp/tmp.aNjXrrl2im ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hymoQIPs1l +++ mktemp ++ local LAST_ERR=/tmp/tmp.Ng6vn8zXwU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.hymoQIPs1l ++ cat /tmp/tmp.Ng6vn8zXwU ++ rm /tmp/tmp.hymoQIPs1l /tmp/tmp.Ng6vn8zXwU ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3cXOXuImeO +++ mktemp ++ local LAST_ERR=/tmp/tmp.ndRT2XjLHI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3cXOXuImeO ++ cat /tmp/tmp.ndRT2XjLHI ++ rm /tmp/tmp.3cXOXuImeO /tmp/tmp.ndRT2XjLHI ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gwPuRr1Zgx +++ mktemp ++ local LAST_ERR=/tmp/tmp.j93GtOVQl1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.gwPuRr1Zgx ++ cat /tmp/tmp.j93GtOVQl1 ++ rm /tmp/tmp.gwPuRr1Zgx /tmp/tmp.j93GtOVQl1 ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mnENO8tCcg +++ mktemp ++ local LAST_ERR=/tmp/tmp.7Sq0PSTXff ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mnENO8tCcg ++ cat /tmp/tmp.7Sq0PSTXff ++ rm /tmp/tmp.mnENO8tCcg /tmp/tmp.7Sq0PSTXff ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 5 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ykSEFRHCc0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.6HYumjr0x6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ykSEFRHCc0 ++ cat /tmp/tmp.6HYumjr0x6 ++ rm /tmp/tmp.ykSEFRHCc0 /tmp/tmp.6HYumjr0x6 ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 6 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dVyU3d0vOZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.LXGdFQCUlk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dVyU3d0vOZ ++ cat /tmp/tmp.LXGdFQCUlk ++ rm /tmp/tmp.dVyU3d0vOZ /tmp/tmp.LXGdFQCUlk ++ return 0 + [[ error == \r\e\a\d\y ]] + let retry+=1 + '[' 7 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nF19eRsvJx +++ mktemp ++ local LAST_ERR=/tmp/tmp.4k8NCCTWOg ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nF19eRsvJx ++ cat /tmp/tmp.4k8NCCTWOg ++ rm /tmp/tmp.nF19eRsvJx /tmp/tmp.4k8NCCTWOg ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 8 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.U5o1HGS6Hl +++ mktemp ++ local LAST_ERR=/tmp/tmp.GGqOblMiCR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.U5o1HGS6Hl ++ cat /tmp/tmp.GGqOblMiCR ++ rm /tmp/tmp.U5o1HGS6Hl /tmp/tmp.GGqOblMiCR ++ return 0 + [[ initializing == \r\e\a\d\y ]] + let retry+=1 + '[' 9 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-new -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Abdttef85x +++ mktemp ++ local LAST_ERR=/tmp/tmp.RZsS3KmS92 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-new -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Abdttef85x ++ cat /tmp/tmp.RZsS3KmS92 ++ rm /tmp/tmp.Abdttef85x /tmp/tmp.RZsS3KmS92 ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + wait_for_pbm_operations some-name-new + local cluster=some-name-new + set +o xtrace waiting for PBM operation to finish.OK + compare_mongo_cmd find myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:54+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296 mongodb '' '' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + local 'command=use myApp\n db.test.find()' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local uri=myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FARGpQc4xi +++ mktemp ++ local LAST_ERR=/tmp/tmp.oKLmQJvCwy ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FARGpQc4xi ++ cat /tmp/tmp.oKLmQJvCwy ++ rm /tmp/tmp.FARGpQc4xi /tmp/tmp.oKLmQJvCwy ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.D2Os5la65U ++ mktemp + local LAST_ERR=/tmp/tmp.rbWbR2yqO2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-0.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.D2Os5la65U + cat /tmp/tmp.rbWbR2yqO2 + rm /tmp/tmp.D2Os5la65U /tmp/tmp.rbWbR2yqO2 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:57+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3NvNMi0mRt +++ mktemp ++ local LAST_ERR=/tmp/tmp.h0EPDmGgrn ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.3NvNMi0mRt ++ cat /tmp/tmp.h0EPDmGgrn ++ rm /tmp/tmp.3NvNMi0mRt /tmp/tmp.h0EPDmGgrn ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.IA0MXysotB ++ mktemp + local LAST_ERR=/tmp/tmp.ioTUg6H7nS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-1.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.IA0MXysotB + cat /tmp/tmp.ioTUg6H7nS + rm /tmp/tmp.IA0MXysotB /tmp/tmp.ioTUg6H7nS + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:38:59+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.v0BZr0qlBD +++ mktemp ++ local LAST_ERR=/tmp/tmp.HHdJFEL59y ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.v0BZr0qlBD ++ cat /tmp/tmp.HHdJFEL59y ++ rm /tmp/tmp.v0BZr0qlBD /tmp/tmp.HHdJFEL59y ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.FKkLgCO1VR ++ mktemp + local LAST_ERR=/tmp/tmp.BwToRVSwmk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-new-rs0-2.some-name-new-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.FKkLgCO1VR + cat /tmp/tmp.BwToRVSwmk + rm /tmp/tmp.FKkLgCO1VR /tmp/tmp.BwToRVSwmk + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + log 'Data restored: OK' + set +o xtrace [2026-03-26T09:39:01+0000] Data restored: OK + echo 'Cleaning up new cluster' Cleaning up new cluster + kubectl_bin delete psmdb some-name-new ++ mktemp + local LAST_OUT=/tmp/tmp.q6vnqOBYOl ++ mktemp + local LAST_ERR=/tmp/tmp.hMkTPMPsDz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb some-name-new + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.q6vnqOBYOl perconaservermongodb.psmdb.percona.com "some-name-new" deleted from demand-backup-snapshot-13296 namespace + cat /tmp/tmp.hMkTPMPsDz + rm /tmp/tmp.q6vnqOBYOl /tmp/tmp.hMkTPMPsDz + return 0 + kubectl_bin wait --for=delete psmdb/some-name-new --timeout=600s ++ mktemp + local LAST_OUT=/tmp/tmp.QNPgjuM0am ++ mktemp + local LAST_ERR=/tmp/tmp.8QjfIuYHjV + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete psmdb/some-name-new --timeout=600s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.QNPgjuM0am + cat /tmp/tmp.8QjfIuYHjV + rm /tmp/tmp.QNPgjuM0am /tmp/tmp.8QjfIuYHjV + return 0 + desc 'Testing snapshot backup/restore on encrypted cluster' + set +o xtrace ----------------------------------------------------------------------------------- Testing snapshot backup/restore on encrypted cluster ----------------------------------------------------------------------------------- + kubectl_bin delete psmdb some-name ++ mktemp + local LAST_OUT=/tmp/tmp.SE6h2t4ZYp ++ mktemp + local LAST_ERR=/tmp/tmp.3EalM5JwDk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete psmdb some-name + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.SE6h2t4ZYp perconaservermongodb.psmdb.percona.com "some-name" deleted from demand-backup-snapshot-13296 namespace + cat /tmp/tmp.3EalM5JwDk + rm /tmp/tmp.SE6h2t4ZYp /tmp/tmp.3EalM5JwDk + return 0 + kubectl_bin wait --for=delete psmdb/some-name --timeout=600s ++ mktemp + local LAST_OUT=/tmp/tmp.lKrS2Hsp9c ++ mktemp + local LAST_ERR=/tmp/tmp.uTFaduK2DD + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete psmdb/some-name --timeout=600s + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.lKrS2Hsp9c + cat /tmp/tmp.uTFaduK2DD + rm /tmp/tmp.lKrS2Hsp9c /tmp/tmp.uTFaduK2DD + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.tGz1hTJ2Xg ++ mktemp + local LAST_ERR=/tmp/tmp.lNYlYm2nxn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tGz1hTJ2Xg secret/some-users configured + cat /tmp/tmp.lNYlYm2nxn Warning: resource secrets/some-users is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.tGz1hTJ2Xg /tmp/tmp.lNYlYm2nxn + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-encrypted.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-encrypted.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/some-name-encrypted.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2271-c31e8c20"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.YfxCNbgtGy + /usr/sbin/sed -e s/NAME_SPACE/demand-backup-snapshot-13296/g ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.RH0lBF5eXm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YfxCNbgtGy perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.RH0lBF5eXm + rm /tmp/tmp.YfxCNbgtGy /tmp/tmp.RH0lBF5eXm + return 0 + echo 'check if all pods started' check if all pods started + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready.OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.roSs2xyQj5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XxqdLjpN5p ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.roSs2xyQj5 ++ cat /tmp/tmp.XxqdLjpN5p ++ rm /tmp/tmp.roSs2xyQj5 /tmp/tmp.XxqdLjpN5p ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JVdauPL3hx +++ mktemp ++ local LAST_ERR=/tmp/tmp.gQ8Tr5EUHL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JVdauPL3hx ++ cat /tmp/tmp.gQ8Tr5EUHL ++ rm /tmp/tmp.JVdauPL3hx /tmp/tmp.gQ8Tr5EUHL ++ return 0 + [[ '' == \t\r\u\e ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qnsOani0uF +++ mktemp ++ local LAST_ERR=/tmp/tmp.TRQieyOBmf ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qnsOani0uF ++ cat /tmp/tmp.TRQieyOBmf ++ rm /tmp/tmp.qnsOani0uF /tmp/tmp.TRQieyOBmf ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness............................................................................. + wait_cluster_consistency some-name + local cluster_name=some-name + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Tjtd01dmPB +++ mktemp ++ local LAST_ERR=/tmp/tmp.HLmWfvmQ9V ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Tjtd01dmPB ++ cat /tmp/tmp.HLmWfvmQ9V ++ rm /tmp/tmp.Tjtd01dmPB /tmp/tmp.HLmWfvmQ9V ++ return 0 + [[ ready == \r\e\a\d\y ]] + echo .OK .OK + sleep 60 + wait_for_pbm_operations some-name + local cluster=some-name + set +o xtrace waiting for PBM operation to finish.OK + echo 'Creating app user and writing test data (encrypted)' Creating app user and writing test data (encrypted) + run_mongo_tls 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TY6ZxKRuSN +++ mktemp ++ local LAST_ERR=/tmp/tmp.fTibO7ve5j ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TY6ZxKRuSN ++ cat /tmp/tmp.fTibO7ve5j ++ rm /tmp/tmp.TY6ZxKRuSN /tmp/tmp.fTibO7ve5j ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.zPuuKU3nmT ++ mktemp + local LAST_ERR=/tmp/tmp.ZNCXUucr8v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zPuuKU3nmT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:43:13.869Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("282a899d-fa71-497f-af41-3e582df7e360") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.ZNCXUucr8v + rm /tmp/tmp.zPuuKU3nmT /tmp/tmp.ZNCXUucr8v + return 0 + sleep 1 + run_mongo_tls 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.23W0EbCqln +++ mktemp ++ local LAST_ERR=/tmp/tmp.wJb4ErVHC6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.23W0EbCqln ++ cat /tmp/tmp.wJb4ErVHC6 ++ rm /tmp/tmp.23W0EbCqln /tmp/tmp.wJb4ErVHC6 ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.YsY8aBW3mc ++ mktemp + local LAST_ERR=/tmp/tmp.fNei1Nzv8h + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.YsY8aBW3mc Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:43:17.553Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("739bcbf0-badb-49ae-a8bb-03a3372c59e6") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.fNei1Nzv8h + rm /tmp/tmp.YsY8aBW3mc /tmp/tmp.fNei1Nzv8h + return 0 + sleep 5 + compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:22+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oZOpRhhY4q +++ mktemp ++ local LAST_ERR=/tmp/tmp.bzCRpczs9A ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.oZOpRhhY4q ++ cat /tmp/tmp.bzCRpczs9A ++ rm /tmp/tmp.oZOpRhhY4q /tmp/tmp.bzCRpczs9A ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.JD6ZZEOKOU ++ mktemp + local LAST_ERR=/tmp/tmp.bgK2YD1jNo + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.JD6ZZEOKOU + cat /tmp/tmp.bgK2YD1jNo + rm /tmp/tmp.JD6ZZEOKOU /tmp/tmp.bgK2YD1jNo + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:25+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HequqMFKc3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.tOc8C3mZHk ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.HequqMFKc3 ++ cat /tmp/tmp.tOc8C3mZHk ++ rm /tmp/tmp.HequqMFKc3 /tmp/tmp.tOc8C3mZHk ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.MgVn3hQYgF ++ mktemp + local LAST_ERR=/tmp/tmp.AjRhls6aku + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MgVn3hQYgF + cat /tmp/tmp.AjRhls6aku + rm /tmp/tmp.MgVn3hQYgF /tmp/tmp.AjRhls6aku + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 '' '' '' '' '' true + local command=find + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=true + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-03-26T09:43:27+0000] running db.test.find() in myApp + [[ true == \t\r\u\e ]] + mongo_command=run_mongo_tls + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + run_mongo_tls 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb + local suffix=.svc.cluster.local + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.W53PtvYc6p +++ mktemp ++ local LAST_ERR=/tmp/tmp.pNHszWDR6w ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.W53PtvYc6p ++ cat /tmp/tmp.pNHszWDR6w ++ rm /tmp/tmp.W53PtvYc6p /tmp/tmp.pNHszWDR6w ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.luioep8emE ++ mktemp + local LAST_ERR=/tmp/tmp.ByyhKuRCO9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.luioep8emE + cat /tmp/tmp.ByyhKuRCO9 + rm /tmp/tmp.luioep8emE /tmp/tmp.ByyhKuRCO9 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/compare/find.json /tmp/tmp.C3dxGR0Bkz/find + echo 'Running snapshot backup (encrypted)' Running snapshot backup (encrypted) + backup_name_enc=backup-snapshot-encrypted + run_snapshot_backup backup-snapshot-encrypted + local backup_name=backup-snapshot-encrypted + log 'running snapshot backup backup-snapshot-encrypted' + set +o xtrace [2026-03-26T09:43:30+0000] running snapshot backup backup-snapshot-encrypted + yq eval '.metadata.name = "backup-snapshot-encrypted" | .spec.volumeSnapshotClass = "gke-snapshot-class"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/backup.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.f2e37D2U9T ++ mktemp + local LAST_ERR=/tmp/tmp.3qXCIZ1cXa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f2e37D2U9T perconaservermongodbbackup.psmdb.percona.com/backup-snapshot-encrypted created + cat /tmp/tmp.3qXCIZ1cXa + rm /tmp/tmp.f2e37D2U9T /tmp/tmp.3qXCIZ1cXa + return 0 + wait_backup backup-snapshot-encrypted + local backup_name=backup-snapshot-encrypted + local target_state=ready + set +o xtrace waiting for backup-snapshot-encrypted to reach ready state......................OK + echo 'Drop collection and restore from snapshot (encrypted)' Drop collection and restore from snapshot (encrypted) + run_mongo_tls 'use myApp\n db.test.drop()' myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 + local driver=mongodb+srv + local suffix=.svc.cluster.local ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IgyXHBguei +++ mktemp ++ local LAST_ERR=/tmp/tmp.b0B3rubxok ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.IgyXHBguei ++ cat /tmp/tmp.b0B3rubxok ++ rm /tmp/tmp.IgyXHBguei /tmp/tmp.b0B3rubxok ++ return 0 + local client_container=psmdb-client-5649fbb65f-bxk9c + local mongo_flag= + local port=27017 ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27017 + [[ myApp:myPass@some-name-rs0.demand-backup-snapshot-13296 == *cfg* ]] + replica_set=rs0 + kubectl_bin exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' ++ mktemp + local LAST_OUT=/tmp/tmp.E6gbHw64xU ++ mktemp + local LAST_ERR=/tmp/tmp.0ae5ybxzqh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-5649fbb65f-bxk9c -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?replicaSet=rs0 --tls --tlsCAFile /etc/mongodb-ssl/ca.crt --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.E6gbHw64xU Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-rs0-0.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.demand-backup-snapshot-13296.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=true {"t":{"$date":"2026-03-26T09:44:17.426Z"},"s":"I", "c":"NETWORK", "id":5490002, "ctx":"thread1","msg":"Started a new thread for the timer service"} Implicit session: session { "id" : UUID("4e6536f9-3bc5-4959-bd81-d774f9146598") } Percona Server for MongoDB server version: v8.0.19-7 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.0ae5ybxzqh + rm /tmp/tmp.E6gbHw64xU /tmp/tmp.0ae5ybxzqh + return 0 + run_restore backup-snapshot-encrypted + local backup_name=backup-snapshot-encrypted + local restore_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/restore.yml + log 'running restore restore-backup-snapshot-encrypted' + set +o xtrace [2026-03-26T09:44:17+0000] running restore restore-backup-snapshot-encrypted + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2271/e2e-tests/demand-backup-snapshot/conf/restore.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-snapshot-encrypted/' + /usr/sbin/sed -e 's/backupName:/backupName: backup-snapshot-encrypted/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.ejb1Ol4xDY ++ mktemp + local LAST_ERR=/tmp/tmp.l5pjKXFckw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ejb1Ol4xDY perconaservermongodbrestore.psmdb.percona.com/restore-backup-snapshot-encrypted created + cat /tmp/tmp.l5pjKXFckw + rm /tmp/tmp.ejb1Ol4xDY /tmp/tmp.l5pjKXFckw + return 0 + run_snapshot_recovery_check backup-snapshot-encrypted + local backup_name=backup-snapshot-encrypted + local target_cluster=some-name + wait_restore backup-snapshot-encrypted some-name ready 0 3000 + local backup_name=backup-snapshot-encrypted + local cluster_name=some-name + local target_state=ready + local wait_cluster_consistency=0 + local wait_time=3000 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-snapshot-encrypted object to be created.OK Waiting psmdb-restore/restore-backup-snapshot-encrypted to reach state "ready" ....