Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/logs/pitr-to-new-cluster.log Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1 + source_cluster=some-name-source + target_cluster=some-name-target + backup_name_minio=backup-minio + setup_infra + create_infra pitr-to-new-cluster-3607 + local ns=pitr-to-new-cluster-3607 + [[ 1 == 1 ]] + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.8A45uBpZ9e ++ mktemp + local LAST_ERR=/tmp/tmp.JT32NDZ96q + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8A45uBpZ9e customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.JT32NDZ96q + rm /tmp/tmp.8A45uBpZ9e /tmp/tmp.JT32NDZ96q + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml ++ grep -v '\-\-\-' grep: warning: stray \ before - grep: warning: stray \ before - + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.tgtNTMSKzt ++ mktemp + local LAST_ERR=/tmp/tmp.anVvQYHJrc + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tgtNTMSKzt customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met + cat /tmp/tmp.anVvQYHJrc + rm /tmp/tmp.tgtNTMSKzt /tmp/tmp.anVvQYHJrc + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.xiKsAjD7ax ++ mktemp + local LAST_ERR=/tmp/tmp.NRowUbybPd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xiKsAjD7ax + cat /tmp/tmp.NRowUbybPd + rm /tmp/tmp.xiKsAjD7ax /tmp/tmp.NRowUbybPd + return 0 + for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-') + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.e2ydG3DwGD ++ mktemp + local LAST_ERR=/tmp/tmp.25AUBNHt4M + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.e2ydG3DwGD + cat /tmp/tmp.25AUBNHt4M + rm /tmp/tmp.e2ydG3DwGD /tmp/tmp.25AUBNHt4M + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.gsOLOSyNVv ++ mktemp + local LAST_ERR=/tmp/tmp.0DESj3kTnn + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.gsOLOSyNVv clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.0DESj3kTnn + rm /tmp/tmp.gsOLOSyNVv /tmp/tmp.0DESj3kTnn + return 0 + check_crd_for_deletion PR-2283-73ca3795 + local git_tag=PR-2283-73ca3795 ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2283-73ca3795/deploy/crd.yaml ++ yq eval .metadata.name ++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/sbin/sed s/---//g + for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g') ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AWitq6W0aX +++ mktemp ++ local LAST_ERR=/tmp/tmp.qPH2xrd9cY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.AWitq6W0aX ++ cat /tmp/tmp.qPH2xrd9cY Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.AWitq6W0aX ++ cat /tmp/tmp.qPH2xrd9cY Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.AWitq6W0aX ++ cat /tmp/tmp.qPH2xrd9cY Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.AWitq6W0aX ++ cat /tmp/tmp.qPH2xrd9cY Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.AWitq6W0aX /tmp/tmp.qPH2xrd9cY ++ return 1 + [[ '' == Terminating ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.XWj3qKfxxA + local LAST_OUT=/tmp/tmp.mx3N0J83j0 ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.iz3wyvVzPj + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.9d9Ui4Foa7 + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mx3N0J83j0 + cat /tmp/tmp.9d9Ui4Foa7 + rm /tmp/tmp.mx3N0J83j0 /tmp/tmp.9d9Ui4Foa7 + return 0 namespace "pitr-to-new-cluster-21867" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XWj3qKfxxA namespace "psmdb-operator" deleted + cat /tmp/tmp.iz3wyvVzPj + rm /tmp/tmp.XWj3qKfxxA /tmp/tmp.iz3wyvVzPj + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.xORmvrtwWb ++ mktemp + local LAST_ERR=/tmp/tmp.Sh9j4m1vnS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xORmvrtwWb + cat /tmp/tmp.Sh9j4m1vnS + rm /tmp/tmp.xORmvrtwWb /tmp/tmp.Sh9j4m1vnS + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.qOp13gz30X ++ mktemp + local LAST_ERR=/tmp/tmp.lnZViFSi5V + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qOp13gz30X namespace/psmdb-operator created + cat /tmp/tmp.lnZViFSi5V + rm /tmp/tmp.qOp13gz30X /tmp/tmp.lnZViFSi5V + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.Z8agrLRx1T +++ mktemp ++ local LAST_ERR=/tmp/tmp.sj5HvzhEnG ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Z8agrLRx1T ++ cat /tmp/tmp.sj5HvzhEnG ++ rm /tmp/tmp.Z8agrLRx1T /tmp/tmp.sj5HvzhEnG ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.f6atSh9c40 ++ mktemp + local LAST_ERR=/tmp/tmp.HUckPxxMed + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f6atSh9c40 Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4" modified. + cat /tmp/tmp.HUckPxxMed + rm /tmp/tmp.f6atSh9c40 /tmp/tmp.HUckPxxMed + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.o3wcGb0Nsr ++ mktemp + local LAST_ERR=/tmp/tmp.BMHWEHb7i5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.o3wcGb0Nsr customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.BMHWEHb7i5 + rm /tmp/tmp.o3wcGb0Nsr /tmp/tmp.BMHWEHb7i5 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.NugWfZi8Lo ++ mktemp + local LAST_ERR=/tmp/tmp.ijMZHxSrg2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NugWfZi8Lo clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.ijMZHxSrg2 + rm /tmp/tmp.NugWfZi8Lo /tmp/tmp.ijMZHxSrg2 + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.taGFoujakm ++ mktemp + local LAST_ERR=/tmp/tmp.tqxm6eChJU + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.taGFoujakm deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.tqxm6eChJU + rm /tmp/tmp.taGFoujakm /tmp/tmp.tqxm6eChJU + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.MqyzKm69jr +++ mktemp ++ local LAST_ERR=/tmp/tmp.UlbDKBemen ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MqyzKm69jr ++ cat /tmp/tmp.UlbDKBemen ++ rm /tmp/tmp.MqyzKm69jr /tmp/tmp.UlbDKBemen ++ return 0 + wait_operator_pod percona-server-mongodb-operator-79d56c85d6-zjl56 + local pod=percona-server-mongodb-operator-79d56c85d6-zjl56 + set +o xtrace waiting for pod/percona-server-mongodb-operator-79d56c85d6-zjl56 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.rISSYGohN7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JGY7yYeukE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rISSYGohN7 ++ cat /tmp/tmp.JGY7yYeukE ++ rm /tmp/tmp.rISSYGohN7 /tmp/tmp.JGY7yYeukE ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-79d56c85d6-zjl56 ++ mktemp + local LAST_OUT=/tmp/tmp.qmu0etlJTz ++ mktemp + local LAST_ERR=/tmp/tmp.i8p4llCrnA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-79d56c85d6-zjl56 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qmu0etlJTz + cat /tmp/tmp.i8p4llCrnA + rm /tmp/tmp.qmu0etlJTz /tmp/tmp.i8p4llCrnA + return 0 2026-04-09T17:00:14.298Z INFO setup Manager starting up {"gitCommit": "73ca37956356b4f3579a3ef1dce0c4105fda2da2", "gitBranch": "PR-2283-73ca3795", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"} + create_namespace pitr-to-new-cluster-3607 + local namespace=pitr-to-new-cluster-3607 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-to-new-cluster-3607' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-to-new-cluster-3607 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pitr-to-new-cluster-3607 --ignore-not-found + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.t1EAbWM3gN + local LAST_OUT=/tmp/tmp.9g5wzJQJqC ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.HhhzepBrJv + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.e8Inv5XRfi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-to-new-cluster-3607 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9g5wzJQJqC + cat /tmp/tmp.HhhzepBrJv + rm /tmp/tmp.9g5wzJQJqC /tmp/tmp.HhhzepBrJv + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.t1EAbWM3gN + cat /tmp/tmp.e8Inv5XRfi + rm /tmp/tmp.t1EAbWM3gN /tmp/tmp.e8Inv5XRfi + return 0 + kubectl_bin wait --for=delete namespace pitr-to-new-cluster-3607 ++ mktemp + local LAST_OUT=/tmp/tmp.ZOIv1q8Rb9 ++ mktemp + local LAST_ERR=/tmp/tmp.fjuelGL5fX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace pitr-to-new-cluster-3607 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZOIv1q8Rb9 + cat /tmp/tmp.fjuelGL5fX + rm /tmp/tmp.ZOIv1q8Rb9 /tmp/tmp.fjuelGL5fX + return 0 + desc 'create namespace pitr-to-new-cluster-3607' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-to-new-cluster-3607 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-to-new-cluster-3607 ++ mktemp + local LAST_OUT=/tmp/tmp.l4XLyhK0z7 ++ mktemp + local LAST_ERR=/tmp/tmp.lBt8elKGy9 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-to-new-cluster-3607 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.l4XLyhK0z7 namespace/pitr-to-new-cluster-3607 created + cat /tmp/tmp.lBt8elKGy9 + rm /tmp/tmp.l4XLyhK0z7 /tmp/tmp.lBt8elKGy9 + return 0 + set_kube_ctx pitr-to-new-cluster-3607 + local namespace=pitr-to-new-cluster-3607 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.QkwMutUGlx +++ mktemp ++ local LAST_ERR=/tmp/tmp.A4zn4VP6U3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QkwMutUGlx ++ cat /tmp/tmp.A4zn4VP6U3 ++ rm /tmp/tmp.QkwMutUGlx /tmp/tmp.A4zn4VP6U3 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4 --namespace=pitr-to-new-cluster-3607 ++ mktemp + local LAST_OUT=/tmp/tmp.ZDaGCFWljw ++ mktemp + local LAST_ERR=/tmp/tmp.VAOuozNge5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4 --namespace=pitr-to-new-cluster-3607 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ZDaGCFWljw Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2283-73ca3795-11-cluster4" modified. + cat /tmp/tmp.VAOuozNge5 + rm /tmp/tmp.ZDaGCFWljw /tmp/tmp.VAOuozNge5 + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Thu Apr 9 17:00:48 2026 NAMESPACE: pitr-to-new-cluster-3607 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-to-new-cluster-3607.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-to-new-cluster-3607 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-to-new-cluster-3607 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-to-new-cluster-3607 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-to-new-cluster-3607 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vlEDvV793B +++ mktemp ++ local LAST_ERR=/tmp/tmp.4dVmxPXBaz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.vlEDvV793B ++ cat /tmp/tmp.4dVmxPXBaz ++ rm /tmp/tmp.vlEDvV793B /tmp/tmp.4dVmxPXBaz ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-96lxs + wait_pod minio-service-6d5f646cdc-96lxs + local pod=minio-service-6d5f646cdc-96lxs + set +o xtrace waiting for pod/minio-service-6d5f646cdc-96lxs to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-to-new-cluster-3607.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.LuGdG0s6b8 ++ mktemp + local LAST_ERR=/tmp/tmp.VNtp4QB4nw + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LuGdG0s6b8 pod "aws-cli" deleted from pitr-to-new-cluster-3607 namespace + cat /tmp/tmp.VNtp4QB4nw All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. + rm /tmp/tmp.LuGdG0s6b8 /tmp/tmp.VNtp4QB4nw + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ykQwEOk4uD ++ mktemp + local LAST_ERR=/tmp/tmp.eLRdht0bOv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.ykQwEOk4uD secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.eLRdht0bOv + rm /tmp/tmp.ykQwEOk4uD /tmp/tmp.eLRdht0bOv + return 0 + setup_source_cluster + desc 'create PSMDB cluster: some-name-source' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster: some-name-source ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-to-new-cluster-3607/g + local LAST_OUT=/tmp/tmp.V3lMhli451 ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.R95TQIfdgN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.V3lMhli451 perconaservermongodb.psmdb.percona.com/some-name-source created + cat /tmp/tmp.R95TQIfdgN + rm /tmp/tmp.V3lMhli451 /tmp/tmp.R95TQIfdgN + return 0 + wait_for_running some-name-source-rs0 3 + local name=some-name-source-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name-source ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-source-rs0-0 + local pod=some-name-source-rs0-0 + set +o xtrace waiting for pod/some-name-source-rs0-0 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-source-rs0-1 + local pod=some-name-source-rs0-1 + set +o xtrace waiting for pod/some-name-source-rs0-1 to be ready......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mL12s6Wesm +++ mktemp ++ local LAST_ERR=/tmp/tmp.SuPJuwTOuW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mL12s6Wesm ++ cat /tmp/tmp.SuPJuwTOuW ++ rm /tmp/tmp.mL12s6Wesm /tmp/tmp.SuPJuwTOuW ++ return 0 + [[ '' == true ]] + wait_pod some-name-source-rs0-2 + local pod=some-name-source-rs0-2 + set +o xtrace waiting for pod/some-name-source-rs0-2 to be ready..............OK ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B4QE3Ggj1M +++ mktemp ++ local LAST_ERR=/tmp/tmp.Gg7xG7xFkS ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.B4QE3Ggj1M ++ cat /tmp/tmp.Gg7xG7xFkS ++ rm /tmp/tmp.B4QE3Ggj1M /tmp/tmp.Gg7xG7xFkS ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0gkHp8uB0M +++ mktemp ++ local LAST_ERR=/tmp/tmp.GvACTQlL0m ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.0gkHp8uB0M ++ cat /tmp/tmp.GvACTQlL0m ++ rm /tmp/tmp.0gkHp8uB0M /tmp/tmp.GvACTQlL0m ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness........ + wait_backup_agent some-name-source-rs0-0 + local agent_pod=some-name-source-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-0...2026-04-09T17:02:47.000+0000 I listening for the commands + wait_backup_agent some-name-source-rs0-1 + local agent_pod=some-name-source-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-1...2026-04-09T17:03:21.000+0000 I listening for the commands + wait_backup_agent some-name-source-rs0-2 + local agent_pod=some-name-source-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-2...2026-04-09T17:03:25.000+0000 I listening for the commands + create_user some-name-source + local cluster=some-name-source + log 'create user myApp' + set +o xtrace [2026-04-09T17:03:37+0000] create user myApp + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ftSDnUNRnD +++ mktemp ++ local LAST_ERR=/tmp/tmp.0iqf1NqJ3K ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ftSDnUNRnD ++ cat /tmp/tmp.0iqf1NqJ3K ++ rm /tmp/tmp.ftSDnUNRnD /tmp/tmp.0iqf1NqJ3K ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.kUCLLIWmDY ++ mktemp + local LAST_ERR=/tmp/tmp.hoPWjzmjfu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kUCLLIWmDY Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("d0fe08c1-c057-4025-811b-54ee7ca7b22a") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.hoPWjzmjfu + rm /tmp/tmp.kUCLLIWmDY /tmp/tmp.hoPWjzmjfu + return 0 + write_document some-name-source 100500 + local cluster=some-name-source + local x=100500 + log 'write document: 100500' + set +o xtrace [2026-04-09T17:03:40+0000] write document: 100500 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.omdJiJZqde +++ mktemp ++ local LAST_ERR=/tmp/tmp.X3J9JirNQc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.omdJiJZqde ++ cat /tmp/tmp.X3J9JirNQc ++ rm /tmp/tmp.omdJiJZqde /tmp/tmp.X3J9JirNQc ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.DgJLWGYuuB ++ mktemp + local LAST_ERR=/tmp/tmp.jxop9WSdXq + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DgJLWGYuuB Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("eb079ebc-47d5-4f32-b9c4-fe158fabc8ef") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.jxop9WSdXq + rm /tmp/tmp.DgJLWGYuuB /tmp/tmp.jxop9WSdXq + return 0 + sleep_with_log 360 'wait for PBM-1265 workaround' + local d=360 + local 'msg=wait for PBM-1265 workaround' + log 'wait for PBM-1265 workaround. sleeping for 360 seconds' + set +o xtrace [2026-04-09T17:03:42+0000] wait for PBM-1265 workaround. sleeping for 360 seconds + sleep 360 + compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local command=find + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-04-09T17:09:42+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4Qq0WxMHni +++ mktemp ++ local LAST_ERR=/tmp/tmp.mdjwUjSMWv ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4Qq0WxMHni ++ cat /tmp/tmp.mdjwUjSMWv ++ rm /tmp/tmp.4Qq0WxMHni /tmp/tmp.mdjwUjSMWv ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.be63ewuFk8 ++ mktemp + local LAST_ERR=/tmp/tmp.wdZn24vZI4 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.be63ewuFk8 + cat /tmp/tmp.wdZn24vZI4 + rm /tmp/tmp.be63ewuFk8 /tmp/tmp.wdZn24vZI4 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/compare/find.json /tmp/tmp.xFOJAcZDQw/find + run_backup some-name-source backup-minio logical + local cluster=some-name-source + local name=backup-minio + local type=logical + desc 'run backup backup-minio-logical' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-logical ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/backup.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-logical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/' + /usr/sbin/sed -e 's/type:/type: logical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.MUhdOXdkOy ++ mktemp + local LAST_ERR=/tmp/tmp.KzTgZdLFzQ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MUhdOXdkOy perconaservermongodbbackup.psmdb.percona.com/backup-minio-logical created + cat /tmp/tmp.KzTgZdLFzQ + rm /tmp/tmp.MUhdOXdkOy /tmp/tmp.KzTgZdLFzQ + return 0 + wait_backup backup-minio-logical + local backup_name=backup-minio-logical + local target_state=ready + set +o xtrace waiting for backup-minio-logical to reach ready state.......OK + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-04-09T17:09:59+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 + run_backup some-name-source backup-minio physical + local cluster=some-name-source + local name=backup-minio + local type=physical + desc 'run backup backup-minio-physical' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-physical ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/backup.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-physical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/' + /usr/sbin/sed -e 's/type:/type: physical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.f6uoJvotfd ++ mktemp + local LAST_ERR=/tmp/tmp.2Y8MFzE91n + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.f6uoJvotfd perconaservermongodbbackup.psmdb.percona.com/backup-minio-physical created + cat /tmp/tmp.2Y8MFzE91n + rm /tmp/tmp.f6uoJvotfd /tmp/tmp.2Y8MFzE91n + return 0 + wait_backup backup-minio-physical + local backup_name=backup-minio-physical + local target_state=ready + set +o xtrace waiting for backup-minio-physical to reach ready state.......OK + desc 'write data for PiTR' + set +o xtrace ----------------------------------------------------------------------------------- write data for PiTR ----------------------------------------------------------------------------------- + write_document some-name-source 100501 + local cluster=some-name-source + local x=100501 + log 'write document: 100501' + set +o xtrace [2026-04-09T17:11:24+0000] write document: 100501 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.feH69WoXko +++ mktemp ++ local LAST_ERR=/tmp/tmp.Z49pasp8Yr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.feH69WoXko ++ cat /tmp/tmp.Z49pasp8Yr ++ rm /tmp/tmp.feH69WoXko /tmp/tmp.Z49pasp8Yr ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.MoCpOVUPvD ++ mktemp + local LAST_ERR=/tmp/tmp.gERepA1Zpv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MoCpOVUPvD Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("81d2d455-f15e-4b2c-90a5-ab97466ffce7") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.gERepA1Zpv + rm /tmp/tmp.MoCpOVUPvD /tmp/tmp.gERepA1Zpv + return 0 + write_document some-name-source 100502 + local cluster=some-name-source + local x=100502 + log 'write document: 100502' + set +o xtrace [2026-04-09T17:11:27+0000] write document: 100502 + run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb + local 'command=use myApp\n db.test.insert({ x: 100502 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.5e6CkrbkB3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lZnkVwyOUJ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5e6CkrbkB3 ++ cat /tmp/tmp.lZnkVwyOUJ ++ rm /tmp/tmp.5e6CkrbkB3 /tmp/tmp.lZnkVwyOUJ ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.hotcoTXC3i ++ mktemp + local LAST_ERR=/tmp/tmp.av5bCZLpd7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hotcoTXC3i Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c1f8c29a-17d7-43a2-aa81-191cb8063306") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.av5bCZLpd7 + rm /tmp/tmp.hotcoTXC3i /tmp/tmp.av5bCZLpd7 + return 0 + write_document some-name-source 100503 + local cluster=some-name-source + local x=100503 + log 'write document: 100503' + set +o xtrace [2026-04-09T17:11:29+0000] write document: 100503 + run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb + local 'command=use myApp\n db.test.insert({ x: 100503 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kHSFV78iHl +++ mktemp ++ local LAST_ERR=/tmp/tmp.vmTDiTEKjx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kHSFV78iHl ++ cat /tmp/tmp.vmTDiTEKjx ++ rm /tmp/tmp.kHSFV78iHl /tmp/tmp.vmTDiTEKjx ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.TIEkh3sETh ++ mktemp + local LAST_ERR=/tmp/tmp.DRFfbIr5Le + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TIEkh3sETh Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("55c1e8fc-54e6-47a4-baf6-c2fd369c6506") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.DRFfbIr5Le + rm /tmp/tmp.TIEkh3sETh /tmp/tmp.DRFfbIr5Le + return 0 + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-04-09T17:11:31+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 + compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-04-09T17:12:41+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YmlroGtJNT +++ mktemp ++ local LAST_ERR=/tmp/tmp.uoAwmcDewR ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.YmlroGtJNT ++ cat /tmp/tmp.uoAwmcDewR ++ rm /tmp/tmp.YmlroGtJNT /tmp/tmp.uoAwmcDewR ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.yL6BggE3hq ++ mktemp + local LAST_ERR=/tmp/tmp.mmXMiObtXC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yL6BggE3hq + cat /tmp/tmp.mmXMiObtXC + rm /tmp/tmp.yL6BggE3hq /tmp/tmp.mmXMiObtXC + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.xFOJAcZDQw/find-2nd + log 'PiTR data is ready: OK' + set +o xtrace [2026-04-09T17:12:44+0000] PiTR data is ready: OK + setup_target_cluster + desc 'create PSMDB cluster: some-name-target' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster: some-name-target ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml ++ mktemp + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2283-73ca3795"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-to-new-cluster-3607/g + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.DBChJUkG1B ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.BncftNnK29 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.DBChJUkG1B perconaservermongodb.psmdb.percona.com/some-name-target created + cat /tmp/tmp.BncftNnK29 + rm /tmp/tmp.DBChJUkG1B /tmp/tmp.BncftNnK29 + return 0 + wait_for_running some-name-target-rs0 3 + local name=some-name-target-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name-target ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-target-rs0-0 + local pod=some-name-target-rs0-0 + set +o xtrace waiting for pod/some-name-target-rs0-0 to be ready................OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-target-rs0-1 + local pod=some-name-target-rs0-1 + set +o xtrace waiting for pod/some-name-target-rs0-1 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.WbzljLbnvT +++ mktemp ++ local LAST_ERR=/tmp/tmp.1b0izuGvSK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.WbzljLbnvT ++ cat /tmp/tmp.1b0izuGvSK ++ rm /tmp/tmp.WbzljLbnvT /tmp/tmp.1b0izuGvSK ++ return 0 + [[ '' == true ]] + wait_pod some-name-target-rs0-2 + local pod=some-name-target-rs0-2 + set +o xtrace waiting for pod/some-name-target-rs0-2 to be ready.....OK ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CL3OZfg5U3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JHzXnAxR9F ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.CL3OZfg5U3 ++ cat /tmp/tmp.JHzXnAxR9F ++ rm /tmp/tmp.CL3OZfg5U3 /tmp/tmp.JHzXnAxR9F ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bZIy6L2urn +++ mktemp ++ local LAST_ERR=/tmp/tmp.R50L4fVZ72 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.bZIy6L2urn ++ cat /tmp/tmp.R50L4fVZ72 ++ rm /tmp/tmp.bZIy6L2urn /tmp/tmp.R50L4fVZ72 ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness.............. + wait_backup_agent some-name-target-rs0-0 + local agent_pod=some-name-target-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-0...2026-04-09T17:14:19.000+0000 I listening for the commands + wait_backup_agent some-name-target-rs0-1 + local agent_pod=some-name-target-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-1...2026-04-09T17:14:56.000+0000 I listening for the commands + wait_backup_agent some-name-target-rs0-2 + local agent_pod=some-name-target-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-2...2026-04-09T17:14:58.000+0000 I listening for the commands + wait_for_pbm_operations some-name-target + local cluster=some-name-target + set +o xtrace waiting for PBM operation to finish.OK + desc 'Case 1: Logical restore' + set +o xtrace ----------------------------------------------------------------------------------- Case 1: Logical restore ----------------------------------------------------------------------------------- + test_case_1 + run_restore some-name-target backup-minio logical + local cluster=some-name-target + local backup_name=backup-minio + local backup_type=logical ++ get_backup_dest backup-minio-logical ++ local backup_name=backup-minio-logical ++ kubectl_bin get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|azure://||' ++ sed 's|s3://||' ++ sed 's|gs://||' +++ mktemp ++ local LAST_OUT=/tmp/tmp.leNtFCnGwZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.VeD4c3jV3N ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.leNtFCnGwZ ++ cat /tmp/tmp.VeD4c3jV3N ++ rm /tmp/tmp.leNtFCnGwZ /tmp/tmp.VeD4c3jV3N ++ return 0 + local destination=operator-testing/some-name-source/2026-04-09T17:09:47Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-logical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/' + /usr/sbin/sed -e 's/type:/type: logical/' + /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2026-04-09T17:09:47Z|' + kubectl_bin apply -f - + /usr/sbin/sed -e s/pitrType:/type:/ ++ mktemp + local LAST_OUT=/tmp/tmp.j9gd0ZPCcs ++ mktemp + local LAST_ERR=/tmp/tmp.rwoodGouPh + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j9gd0ZPCcs perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-logical created + cat /tmp/tmp.rwoodGouPh + rm /tmp/tmp.j9gd0ZPCcs /tmp/tmp.rwoodGouPh + return 0 + wait_restore backup-minio-logical some-name-target requested 0 + local backup_name=backup-minio-logical + local cluster_name=some-name-target + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK Waiting psmdb-restore/restore-backup-minio-logical to reach state "requested" .OK after 0 minutes + [[ 0 -eq 1 ]] + compare_pbm_config some-name-target source + local cluster=some-name-target + local compare_suffix=source + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.iaQzaGNoOs ++ mktemp + local LAST_ERR=/tmp/tmp.h1SSp5H6DJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.iaQzaGNoOs + cat /tmp/tmp.h1SSp5H6DJ + rm /tmp/tmp.iaQzaGNoOs /tmp/tmp.h1SSp5H6DJ + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/compare/pbm_config_source.yml /tmp/tmp.xFOJAcZDQw/pbm_config.yml + log 'Backup source is set as main storage during restore: OK' + set +o xtrace [2026-04-09T17:15:14+0000] Backup source is set as main storage during restore: OK ++ date +%s + local now_ts=1775754914 + wait_restore backup-minio-logical some-name-target ready + local backup_name=backup-minio-logical + local cluster_name=some-name-target + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK Waiting psmdb-restore/restore-backup-minio-logical to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name-target + local cluster_name=some-name-target + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fzmrv4IMoj +++ mktemp ++ local LAST_ERR=/tmp/tmp.bLFQffMw3v ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fzmrv4IMoj ++ cat /tmp/tmp.bLFQffMw3v ++ rm /tmp/tmp.Fzmrv4IMoj /tmp/tmp.bLFQffMw3v ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_pbm_config some-name-target target + local cluster=some-name-target + local compare_suffix=target + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.fWZSng1FFZ ++ mktemp + local LAST_ERR=/tmp/tmp.DGmBiD3ZfK + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.fWZSng1FFZ + cat /tmp/tmp.DGmBiD3ZfK + rm /tmp/tmp.fWZSng1FFZ /tmp/tmp.DGmBiD3ZfK + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/compare/pbm_config_target.yml /tmp/tmp.xFOJAcZDQw/pbm_config.yml + log 'Main storage is reverted after backup-minio-logical: OK' + set +o xtrace [2026-04-09T17:15:32+0000] Main storage is reverted after backup-minio-logical: OK + wait_for_pbm_operations some-name-target + local cluster=some-name-target + set +o xtrace waiting for PBM operation to finish.OK + check_if_resync_triggered some-name-target 1775754914 + local cluster=some-name-target + local ts=1775754914 + log 'checking if resync is triggered after 1775754914' + set +o xtrace [2026-04-09T17:15:34+0000] checking if resync is triggered after 1775754914 ++ run_mongo 'use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})' backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607 ++ local 'command=use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})' ++ local uri=backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local replica_set=rs0 ++ [[ backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FgiJ2lhrwJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3ptC0sGLTp +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.FgiJ2lhrwJ +++ cat /tmp/tmp.3ptC0sGLTp +++ rm /tmp/tmp.FgiJ2lhrwJ /tmp/tmp.3ptC0sGLTp +++ return 0 ++ local client_container=psmdb-client-bb8b97679-4zdcb ++ kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QIkBOHX3Jn +++ mktemp ++ local LAST_ERR=/tmp/tmp.xFiNsFWSPz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.QIkBOHX3Jn ++ cat /tmp/tmp.xFiNsFWSPz command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.QIkBOHX3Jn ++ cat /tmp/tmp.xFiNsFWSPz command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1775754914 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.QIkBOHX3Jn ++ cat /tmp/tmp.xFiNsFWSPz command terminated with exit code 1 ++ sleep 8 ++ cat /tmp/tmp.QIkBOHX3Jn ++ cat /tmp/tmp.xFiNsFWSPz command terminated with exit code 1 ++ rm /tmp/tmp.QIkBOHX3Jn /tmp/tmp.xFiNsFWSPz ++ return 1 + local $'count=Percona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("0167356a-57a2-4811-9f1d-1e872618502a") }\nPercona Server for MongoDB server version: v8.0.20-8\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("7ba0061b-ae9a-4f7f-8394-a2dea7ee3df7") }\nPercona Server for MongoDB server version: v8.0.20-8\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") }\nPercona Server for MongoDB server version: v8.0.20-8\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") }\nPercona Server for MongoDB server version: v8.0.20-8\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye' + [[ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0167356a-57a2-4811-9f1d-1e872618502a") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7ba0061b-ae9a-4f7f-8394-a2dea7ee3df7") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye -lt 1 ]] ./e2e-tests/pitr-to-new-cluster/run: line 91: [[: Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0167356a-57a2-4811-9f1d-1e872618502a") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7ba0061b-ae9a-4f7f-8394-a2dea7ee3df7") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye: arithmetic syntax error: invalid arithmetic operator (error token is ".4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0167356a-57a2-4811-9f1d-1e872618502a") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("7ba0061b-ae9a-4f7f-8394-a2dea7ee3df7") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("ae9cba4f-49a4-41a2-807a-24d88d4109db") } Percona Server for MongoDB server version: v8.0.20-8 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye") + compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-04-09T17:15:52+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607 + local driver=mongodb + local suffix=.svc.cluster.local + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mBTCyPbhuu +++ mktemp ++ local LAST_ERR=/tmp/tmp.Q2Xxe7i3T3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.mBTCyPbhuu ++ cat /tmp/tmp.Q2Xxe7i3T3 ++ rm /tmp/tmp.mBTCyPbhuu /tmp/tmp.Q2Xxe7i3T3 ++ return 0 + local client_container=psmdb-client-bb8b97679-4zdcb + kubectl_bin exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.msQFHdO0bC ++ mktemp + local LAST_ERR=/tmp/tmp.9wpxxpV6yH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-4zdcb -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-3607.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.msQFHdO0bC + cat /tmp/tmp.9wpxxpV6yH + rm /tmp/tmp.msQFHdO0bC /tmp/tmp.9wpxxpV6yH + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.xFOJAcZDQw/find-2nd + log 'Data is restored from backup-minio-logical: OK' + set +o xtrace [2026-04-09T17:15:55+0000] Data is restored from backup-minio-logical: OK + desc 'Case 1: Logical restore: OK' + set +o xtrace ----------------------------------------------------------------------------------- Case 1: Logical restore: OK ----------------------------------------------------------------------------------- + desc 'Case 2: Physical restore' + set +o xtrace ----------------------------------------------------------------------------------- Case 2: Physical restore ----------------------------------------------------------------------------------- + test_case_2 + run_restore some-name-target backup-minio physical + local cluster=some-name-target + local backup_name=backup-minio + local backup_type=physical ++ get_backup_dest backup-minio-physical ++ local backup_name=backup-minio-physical ++ kubectl_bin get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.KeZN5jfc1I +++ mktemp ++ local LAST_ERR=/tmp/tmp.QQsN1eznrY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.KeZN5jfc1I ++ cat /tmp/tmp.QQsN1eznrY ++ rm /tmp/tmp.KeZN5jfc1I /tmp/tmp.QQsN1eznrY ++ return 0 + local destination=operator-testing/some-name-source/2026-04-09T17:11:12Z + /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2026-04-09T17:11:12Z|' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2283/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml + kubectl_bin apply -f - + /usr/sbin/sed -e s/pitrType:/type:/ ++ mktemp + /usr/sbin/sed -e 's/type:/type: physical/' + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-physical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/' + local LAST_OUT=/tmp/tmp.L3LRuQYss2 ++ mktemp + local LAST_ERR=/tmp/tmp.WxkM2IF2q1 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L3LRuQYss2 perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-physical created + cat /tmp/tmp.WxkM2IF2q1 + rm /tmp/tmp.L3LRuQYss2 /tmp/tmp.WxkM2IF2q1 + return 0 + wait_restore backup-minio-physical some-name-target requested 0 + local backup_name=backup-minio-physical + local cluster_name=some-name-target + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-physical object to be created.OK Waiting psmdb-restore/restore-backup-minio-physical to reach state "requested" ....OK after 3 minutes + [[ 0 -eq 1 ]] + compare_pbm_config some-name-target source mongod + local cluster=some-name-target + local compare_suffix=source + local container=mongod + local pbm_binary=pbm + [[ mongod == mongod ]] + pbm_binary=/opt/percona/pbm + kubectl_bin exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.FdaqgTH8qI ++ mktemp + local LAST_ERR=/tmp/tmp.BkN2TFGMte + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.FdaqgTH8qI + cat /tmp/tmp.BkN2TFGMte error: Internal error occurred: Internal error occurred: error executing command in container: failed to exec in container: failed to create exec "7db120c4973e0582f6381e4f2714ce71ee3d896b508b84698edcdbf94959a091": task b8065232e035e18e237c17b923dbbae391a18122fa6bc61a8bdd03bbf02e512e not found: not found + sleep 0 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.FdaqgTH8qI + cat /tmp/tmp.BkN2TFGMte Error from server (NotFound): pods "some-name-target-rs0-0" not found + sleep 4 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config + exit_status=1 + set -e + '[' 1 '!=' 0 -a -n 1 ']' + cat /tmp/tmp.FdaqgTH8qI + cat /tmp/tmp.BkN2TFGMte error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + sleep 8 + cat /tmp/tmp.FdaqgTH8qI + cat /tmp/tmp.BkN2TFGMte error: Internal error occurred: unable to upgrade connection: container not found ("mongod") + rm /tmp/tmp.FdaqgTH8qI /tmp/tmp.BkN2TFGMte + return 1