Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/logs/pitr-to-new-cluster.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + source_cluster=some-name-source + target_cluster=some-name-target + backup_name_minio=backup-minio + setup_infra + create_infra pitr-to-new-cluster-17154 + local ns=pitr-to-new-cluster-17154 + echo pitr-to-new-cluster-17154 + [[ 0 == 1 ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrolebinding ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' ++ mktemp + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.xFoLroYmcm ++ mktemp + local LAST_OUT=/tmp/tmp.2kyHDfGmTy ++ mktemp + local LAST_ERR=/tmp/tmp.MeFNjHfNus + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.L0iXcn5WGS + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + for i in $(seq 0 2) + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.xFoLroYmcm + cat /tmp/tmp.MeFNjHfNus + rm /tmp/tmp.xFoLroYmcm /tmp/tmp.MeFNjHfNus + return 0 namespace "one-pod-22984" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2kyHDfGmTy namespace "psmdb-operator" deleted + cat /tmp/tmp.L0iXcn5WGS + rm /tmp/tmp.2kyHDfGmTy /tmp/tmp.L0iXcn5WGS + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mBD2ANnnWd ++ mktemp + local LAST_ERR=/tmp/tmp.LKRSolVz7N + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mBD2ANnnWd + cat /tmp/tmp.LKRSolVz7N + rm /tmp/tmp.mBD2ANnnWd /tmp/tmp.LKRSolVz7N + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.jJOzgEJbLk ++ mktemp + local LAST_ERR=/tmp/tmp.2L2Wpi0Wt8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jJOzgEJbLk namespace/psmdb-operator created + cat /tmp/tmp.2L2Wpi0Wt8 + rm /tmp/tmp.jJOzgEJbLk /tmp/tmp.2L2Wpi0Wt8 + return 0 + set_kube_ctx psmdb-operator + local namespace=psmdb-operator ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.25cJqRqSxs +++ mktemp ++ local LAST_ERR=/tmp/tmp.AlQIJPNbVU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.25cJqRqSxs ++ cat /tmp/tmp.AlQIJPNbVU ++ rm /tmp/tmp.25cJqRqSxs /tmp/tmp.AlQIJPNbVU ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.vwveaSL6YT ++ mktemp + local LAST_ERR=/tmp/tmp.udiAV1XKZi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vwveaSL6YT Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11" modified. + cat /tmp/tmp.udiAV1XKZi + rm /tmp/tmp.vwveaSL6YT /tmp/tmp.udiAV1XKZi + return 0 + deploy_operator + desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187 ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.RohJwyxgFe ++ mktemp + local LAST_ERR=/tmp/tmp.TiIdfgzWQ2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RohJwyxgFe customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.TiIdfgzWQ2 + rm /tmp/tmp.RohJwyxgFe /tmp/tmp.TiIdfgzWQ2 + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: psmdb-operator^' + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.j6etXqCn3M ++ mktemp + local LAST_ERR=/tmp/tmp.S5zbqkIrQp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.j6etXqCn3M clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator unchanged serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator unchanged + cat /tmp/tmp.S5zbqkIrQp + rm /tmp/tmp.j6etXqCn3M /tmp/tmp.S5zbqkIrQp + return 0 + yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/deploy/cw-operator.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.AVciPtK4TM ++ mktemp + local LAST_ERR=/tmp/tmp.b0SYsMxEuJ + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AVciPtK4TM deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.b0SYsMxEuJ + rm /tmp/tmp.AVciPtK4TM /tmp/tmp.b0SYsMxEuJ + return 0 + sleep 20 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Htvqc7CWel +++ mktemp ++ local LAST_ERR=/tmp/tmp.lngAsDuRoo ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Htvqc7CWel ++ cat /tmp/tmp.lngAsDuRoo ++ rm /tmp/tmp.Htvqc7CWel /tmp/tmp.lngAsDuRoo ++ return 0 + wait_operator_pod percona-server-mongodb-operator-859f4bc6d7-8lkl7 + local pod=percona-server-mongodb-operator-859f4bc6d7-8lkl7 + set +o xtrace waiting for pod/percona-server-mongodb-operator-859f4bc6d7-8lkl7 to be ready.OK + echo 'Print operator info from log' Print operator info from log + grep 'Manager starting up' ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.qJopDMSfuN +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mn3mGR4QXW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qJopDMSfuN ++ cat /tmp/tmp.Mn3mGR4QXW ++ rm /tmp/tmp.qJopDMSfuN /tmp/tmp.Mn3mGR4QXW ++ return 0 + kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-8lkl7 ++ mktemp + local LAST_OUT=/tmp/tmp.WFwuISCu3Z ++ mktemp + local LAST_ERR=/tmp/tmp.Nal99nsNew + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n psmdb-operator percona-server-mongodb-operator-859f4bc6d7-8lkl7 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.WFwuISCu3Z + cat /tmp/tmp.Nal99nsNew + rm /tmp/tmp.WFwuISCu3Z /tmp/tmp.Nal99nsNew + return 0 2026-06-09T09:28:59.736Z INFO setup Manager starting up {"gitCommit": "fb8cc718776dc8f9e011cd5f23fe48cf7f65143e", "gitBranch": "PR-2058-fb8cc7187", "buildTime": "", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} + create_namespace pitr-to-new-cluster-17154 + local namespace=pitr-to-new-cluster-17154 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces pitr-to-new-cluster-17154' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pitr-to-new-cluster-17154 ----------------------------------------------------------------------------------- ++ mktemp + kubectl_bin delete namespace pitr-to-new-cluster-17154 --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.0fWXZ6skm0 ++ mktemp + local LAST_OUT=/tmp/tmp.vwifGI1TAv + local LAST_ERR=/tmp/tmp.mf4IJSQ4O6 ++ mktemp + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Ka2zG3VGsg + local exit_status=0 + local timeout=4 + for i in $(seq 0 2) + set +e + kubectl get ns ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pitr-to-new-cluster-17154 --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.0fWXZ6skm0 + cat /tmp/tmp.mf4IJSQ4O6 + rm /tmp/tmp.0fWXZ6skm0 /tmp/tmp.mf4IJSQ4O6 + return 0 error: resource(s) were provided, but no name was specified + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vwifGI1TAv + cat /tmp/tmp.Ka2zG3VGsg + rm /tmp/tmp.vwifGI1TAv /tmp/tmp.Ka2zG3VGsg + return 0 + kubectl_bin wait --for=delete namespace pitr-to-new-cluster-17154 ++ mktemp + local LAST_OUT=/tmp/tmp.zuyv5V57Pf ++ mktemp + local LAST_ERR=/tmp/tmp.q0dkalYGI0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=delete namespace pitr-to-new-cluster-17154 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zuyv5V57Pf + cat /tmp/tmp.q0dkalYGI0 + rm /tmp/tmp.zuyv5V57Pf /tmp/tmp.q0dkalYGI0 + return 0 + desc 'create namespace pitr-to-new-cluster-17154' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pitr-to-new-cluster-17154 ----------------------------------------------------------------------------------- + kubectl_bin create namespace pitr-to-new-cluster-17154 ++ mktemp + local LAST_OUT=/tmp/tmp.nlvJXm35b7 ++ mktemp + local LAST_ERR=/tmp/tmp.t6i9jLkADM + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pitr-to-new-cluster-17154 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nlvJXm35b7 namespace/pitr-to-new-cluster-17154 created + cat /tmp/tmp.t6i9jLkADM + rm /tmp/tmp.nlvJXm35b7 /tmp/tmp.t6i9jLkADM + return 0 + set_kube_ctx pitr-to-new-cluster-17154 + local namespace=pitr-to-new-cluster-17154 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.n9Z0WX7Zdg +++ mktemp ++ local LAST_ERR=/tmp/tmp.3tm4qyxrpb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.n9Z0WX7Zdg ++ cat /tmp/tmp.3tm4qyxrpb ++ rm /tmp/tmp.n9Z0WX7Zdg /tmp/tmp.3tm4qyxrpb ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=pitr-to-new-cluster-17154 ++ mktemp + local LAST_OUT=/tmp/tmp.8hQNyYQr0T ++ mktemp + local LAST_ERR=/tmp/tmp.bN9uqlGDey + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11 --namespace=pitr-to-new-cluster-17154 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8hQNyYQr0T Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2058-fb8cc7187-17-cluster11" modified. + cat /tmp/tmp.bN9uqlGDey + rm /tmp/tmp.8hQNyYQr0T /tmp/tmp.bN9uqlGDey + return 0 + deploy_minio + local cert_secret= + local service_name=minio-service + desc 'install MinIO: minio-service' + set +o xtrace ----------------------------------------------------------------------------------- install MinIO: minio-service ----------------------------------------------------------------------------------- + helm uninstall minio-service + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + local endpoint=http://minio-service:9000 + minio_args=('--version' '5.4.0' '--set' 'replicas=1' '--set' 'mode=standalone' '--set' 'resources.requests.memory=256Mi' '--set' 'rootUser=rootuser' '--set' 'rootPassword=rootpass123' '--set' 'users[0].accessKey=some-access-key' '--set' 'users[0].secretKey=some-secret-key' '--set' 'users[0].policy=consoleAdmin' '--set' 'service.type=ClusterIP' '--set' 'configPathmc=/tmp/' '--set' 'securityContext.enabled=false' '--set' 'persistence.size=2G' '--set' 'fullnameOverride=minio-service' '--set' 'serviceAccount.create=true' '--set' 'serviceAccount.name=minio-service-sa') + local minio_args + [[ -n '' ]] + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set fullnameOverride=minio-service --set serviceAccount.create=true --set serviceAccount.name=minio-service-sa minio/minio NAME: minio-service LAST DEPLOYED: Tue Jun 9 09:29:35 2026 NAMESPACE: pitr-to-new-cluster-17154 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.pitr-to-new-cluster-17154.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace pitr-to-new-cluster-17154 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace pitr-to-new-cluster-17154 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-to-new-cluster-17154 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-to-new-cluster-17154 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EbX37VvWJJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.bpS2DNyWwQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.EbX37VvWJJ ++ cat /tmp/tmp.bpS2DNyWwQ ++ rm /tmp/tmp.EbX37VvWJJ /tmp/tmp.bpS2DNyWwQ ++ return 0 + local MINIO_POD=minio-service-6d5f646cdc-hgzpj + wait_pod minio-service-6d5f646cdc-hgzpj + local pod=minio-service-6d5f646cdc-hgzpj + set +o xtrace waiting for pod/minio-service-6d5f646cdc-hgzpj to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-to-new-cluster-17154.svc.cluster.local --tcp=9000 service/minio-service created + create_minio_bucket operator-testing http://minio-service:9000 + local bucket=operator-testing + local endpoint=http://minio-service:9000 + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.1057kqZcX3 ++ mktemp + local LAST_ERR=/tmp/tmp.ZlH323uXso + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- bash -c $'AWS_ACCESS_KEY_ID=some-access-key \t\tAWS_SECRET_ACCESS_KEY=some-secret-key \t\tAWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.1057kqZcX3 make_bucket: operator-testing make_bucket: operator-testing pod "aws-cli" deleted from pitr-to-new-cluster-17154 namespace + cat /tmp/tmp.ZlH323uXso All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-to-new-cluster-17154 + rm /tmp/tmp.1057kqZcX3 /tmp/tmp.ZlH323uXso + return 0 + desc 'create secrets and start client' + set +o xtrace ----------------------------------------------------------------------------------- create secrets and start client ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.taUSlOlHJn ++ mktemp + local LAST_ERR=/tmp/tmp.3jaG4YVhWu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/conf/minio-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.taUSlOlHJn secret/some-users created deployment.apps/psmdb-client created secret/minio-secret created + cat /tmp/tmp.3jaG4YVhWu + rm /tmp/tmp.taUSlOlHJn /tmp/tmp.3jaG4YVhWu + return 0 + setup_source_cluster + desc 'create PSMDB cluster: some-name-source' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster: some-name-source ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + kubectl_bin apply -f - ++ mktemp + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-source.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.LNhmgfRBbD ++ mktemp + yq eval '.spec.upgradeOptions.apply="Never"' + /usr/sbin/sed -e s/NAME_SPACE/pitr-to-new-cluster-17154/g + local LAST_ERR=/tmp/tmp.FnquMvL9Ma + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.LNhmgfRBbD perconaservermongodb.psmdb.percona.com/some-name-source created + cat /tmp/tmp.FnquMvL9Ma + rm /tmp/tmp.LNhmgfRBbD /tmp/tmp.FnquMvL9Ma + return 0 + wait_for_running some-name-source-rs0 3 + local name=some-name-source-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name-source ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-source-rs0-0 + local pod=some-name-source-rs0-0 + set +o xtrace waiting for pod/some-name-source-rs0-0 to be ready........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-source-rs0-1 + local pod=some-name-source-rs0-1 + set +o xtrace waiting for pod/some-name-source-rs0-1 to be ready.......OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JPaJr5g16m +++ mktemp ++ local LAST_ERR=/tmp/tmp.IldLDKTroA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.JPaJr5g16m ++ cat /tmp/tmp.IldLDKTroA ++ rm /tmp/tmp.JPaJr5g16m /tmp/tmp.IldLDKTroA ++ return 0 + [[ '' == true ]] + wait_pod some-name-source-rs0-2 + local pod=some-name-source-rs0-2 + set +o xtrace waiting for pod/some-name-source-rs0-2 to be ready......OK ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MchKjWwgda +++ mktemp ++ local LAST_ERR=/tmp/tmp.rBC3PqDgc7 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MchKjWwgda ++ cat /tmp/tmp.rBC3PqDgc7 ++ rm /tmp/tmp.MchKjWwgda /tmp/tmp.rBC3PqDgc7 ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FVan4pdNSZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.TOxnhPsqAK ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-source -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.FVan4pdNSZ ++ cat /tmp/tmp.TOxnhPsqAK ++ rm /tmp/tmp.FVan4pdNSZ /tmp/tmp.TOxnhPsqAK ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness............. + wait_backup_agent some-name-source-rs0-0 + local agent_pod=some-name-source-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-0...2026-06-09T09:31:37.000+0000 I listening for the commands + wait_backup_agent some-name-source-rs0-1 + local agent_pod=some-name-source-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-1...2026-06-09T09:32:11.000+0000 I listening for the commands + wait_backup_agent some-name-source-rs0-2 + local agent_pod=some-name-source-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-source-rs0-2...2026-06-09T09:32:13.000+0000 I listening for the commands + create_user some-name-source + local cluster=some-name-source + log 'create user myApp' + set +o xtrace [2026-06-09T09:32:16+0000] create user myApp + run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kcUrdD4gdv +++ mktemp ++ local LAST_ERR=/tmp/tmp.7kmpy3nxPM ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.kcUrdD4gdv ++ cat /tmp/tmp.7kmpy3nxPM ++ rm /tmp/tmp.kcUrdD4gdv /tmp/tmp.7kmpy3nxPM ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.a328zFuEqE ++ mktemp + local LAST_ERR=/tmp/tmp.xpe3MwlFzm + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.a328zFuEqE Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("4a7e0015-b799-44ac-90af-32914562f06b") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" } ] } bye + cat /tmp/tmp.xpe3MwlFzm + rm /tmp/tmp.a328zFuEqE /tmp/tmp.xpe3MwlFzm + return 0 + write_document some-name-source 100500 + local cluster=some-name-source + local x=100500 + log 'write document: 100500' + set +o xtrace [2026-06-09T09:32:19+0000] write document: 100500 + run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2XeQFtjZ34 +++ mktemp ++ local LAST_ERR=/tmp/tmp.WY3acIUXX1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.2XeQFtjZ34 ++ cat /tmp/tmp.WY3acIUXX1 ++ rm /tmp/tmp.2XeQFtjZ34 /tmp/tmp.WY3acIUXX1 ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Jb9GcXrZlH ++ mktemp + local LAST_ERR=/tmp/tmp.GQi1yfVdPr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Jb9GcXrZlH Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("d48f3702-edf1-4cd9-8994-3b9ed317d2a7") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.GQi1yfVdPr + rm /tmp/tmp.Jb9GcXrZlH /tmp/tmp.GQi1yfVdPr + return 0 + sleep_with_log 360 'wait for PBM-1265 workaround' + local d=360 + local 'msg=wait for PBM-1265 workaround' + log 'wait for PBM-1265 workaround. sleeping for 360 seconds' + set +o xtrace [2026-06-09T09:32:21+0000] wait for PBM-1265 workaround. sleeping for 360 seconds + sleep 360 + compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local command=find + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local postfix= + local suffix= + local database=myApp + local collection=test + local sort= + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n '' ]] + log 'running db.test.find() in myApp' + set +o xtrace [2026-06-09T09:38:21+0000] running db.test.find() in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb '' '' + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rmAezWHPRj +++ mktemp ++ local LAST_ERR=/tmp/tmp.OM2KmrPr4O ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.rmAezWHPRj ++ cat /tmp/tmp.OM2KmrPr4O ++ rm /tmp/tmp.rmAezWHPRj /tmp/tmp.OM2KmrPr4O ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.RaJF8SdUrU ++ mktemp + local LAST_ERR=/tmp/tmp.0nLLCTq1ew + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RaJF8SdUrU + cat /tmp/tmp.0nLLCTq1ew + rm /tmp/tmp.RaJF8SdUrU /tmp/tmp.0nLLCTq1ew + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find.json /tmp/tmp.KlgUW3jDu0/find + run_backup some-name-source backup-minio logical + local cluster=some-name-source + local name=backup-minio + local type=logical + desc 'run backup backup-minio-logical' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-logical ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/backup.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-logical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/' + /usr/sbin/sed -e 's/type:/type: logical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kV932f0gIi ++ mktemp + local LAST_ERR=/tmp/tmp.w7kEkB4qUi + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kV932f0gIi perconaservermongodbbackup.psmdb.percona.com/backup-minio-logical created + cat /tmp/tmp.w7kEkB4qUi + rm /tmp/tmp.kV932f0gIi /tmp/tmp.w7kEkB4qUi + return 0 + wait_backup backup-minio-logical + local backup_name=backup-minio-logical + local target_state=ready + set +o xtrace waiting for backup-minio-logical to reach ready state.......OK + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-06-09T09:38:39+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 + run_backup some-name-source backup-minio physical + local cluster=some-name-source + local name=backup-minio + local type=physical + desc 'run backup backup-minio-physical' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio-physical ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/backup.yml + /usr/sbin/sed -e 's/name:/name: backup-minio-physical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-source/' + kubectl_bin apply -f - + /usr/sbin/sed -e 's/type:/type: physical/' ++ mktemp + local LAST_OUT=/tmp/tmp.riVGDibQRN ++ mktemp + local LAST_ERR=/tmp/tmp.Zrvw2SIF5j + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.riVGDibQRN perconaservermongodbbackup.psmdb.percona.com/backup-minio-physical created + cat /tmp/tmp.Zrvw2SIF5j + rm /tmp/tmp.riVGDibQRN /tmp/tmp.Zrvw2SIF5j + return 0 + wait_backup backup-minio-physical + local backup_name=backup-minio-physical + local target_state=ready + set +o xtrace waiting for backup-minio-physical to reach ready state.......OK + desc 'write data for PiTR' + set +o xtrace ----------------------------------------------------------------------------------- write data for PiTR ----------------------------------------------------------------------------------- + write_document some-name-source 100501 + local cluster=some-name-source + local x=100501 + log 'write document: 100501' + set +o xtrace [2026-06-09T09:40:02+0000] write document: 100501 + run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100501 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p5Y6F9iH9M +++ mktemp ++ local LAST_ERR=/tmp/tmp.YhV3UnHwew ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.p5Y6F9iH9M ++ cat /tmp/tmp.YhV3UnHwew ++ rm /tmp/tmp.p5Y6F9iH9M /tmp/tmp.YhV3UnHwew ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.9vcmPDh9Wn ++ mktemp + local LAST_ERR=/tmp/tmp.NEFOljKBfA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.9vcmPDh9Wn Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("dc411859-1a91-41ea-ace1-306294447cfe") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.NEFOljKBfA + rm /tmp/tmp.9vcmPDh9Wn /tmp/tmp.NEFOljKBfA + return 0 + write_document some-name-source 100502 + local cluster=some-name-source + local x=100502 + log 'write document: 100502' + set +o xtrace [2026-06-09T09:40:05+0000] write document: 100502 + run_mongo 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100502 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9Dn5I6j7Sk +++ mktemp ++ local LAST_ERR=/tmp/tmp.dp5t83GPEb ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9Dn5I6j7Sk ++ cat /tmp/tmp.dp5t83GPEb ++ rm /tmp/tmp.9Dn5I6j7Sk /tmp/tmp.dp5t83GPEb ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.S3pvXUp80r ++ mktemp + local LAST_ERR=/tmp/tmp.FoJd63Sqgs + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.S3pvXUp80r Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("4da795c3-b9d2-48d6-aa69-8adbe1f05167") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.FoJd63Sqgs + rm /tmp/tmp.S3pvXUp80r /tmp/tmp.FoJd63Sqgs + return 0 + write_document some-name-source 100503 + local cluster=some-name-source + local x=100503 + log 'write document: 100503' + set +o xtrace [2026-06-09T09:40:07+0000] write document: 100503 + run_mongo 'use myApp\n db.test.insert({ x: 100503 })' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100503 })' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eFVtG2eqJj +++ mktemp ++ local LAST_ERR=/tmp/tmp.rj2KyWwxkx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.eFVtG2eqJj ++ cat /tmp/tmp.rj2KyWwxkx ++ rm /tmp/tmp.eFVtG2eqJj /tmp/tmp.rj2KyWwxkx ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.TSfyvYfemj ++ mktemp + local LAST_ERR=/tmp/tmp.0ipVQLZnZY + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100503 })\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.TSfyvYfemj Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("0d32fff8-5d32-4084-ada0-d648f2edfb13") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.0ipVQLZnZY + rm /tmp/tmp.TSfyvYfemj /tmp/tmp.0ipVQLZnZY + return 0 + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-06-09T09:40:10+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 + compare_mongo_cmd find myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-06-09T09:41:20+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9T0eouFKtk +++ mktemp ++ local LAST_ERR=/tmp/tmp.6GilM5R7h8 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.9T0eouFKtk ++ cat /tmp/tmp.6GilM5R7h8 ++ rm /tmp/tmp.9T0eouFKtk /tmp/tmp.6GilM5R7h8 ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.aIqLQJFprA ++ mktemp + local LAST_ERR=/tmp/tmp.rkF9SWGzmR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-source-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.aIqLQJFprA + cat /tmp/tmp.rkF9SWGzmR + rm /tmp/tmp.aIqLQJFprA /tmp/tmp.rkF9SWGzmR + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.KlgUW3jDu0/find-2nd + log 'PiTR data is ready: OK' + set +o xtrace [2026-06-09T09:41:22+0000] PiTR data is ready: OK + setup_target_cluster + desc 'create PSMDB cluster: some-name-target' + set +o xtrace ----------------------------------------------------------------------------------- create PSMDB cluster: some-name-target ----------------------------------------------------------------------------------- + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/some-name-target.yml + yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"' ++ mktemp + yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"' + yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2058-fb8cc7187"' + yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.4ftNbTFNBC ++ mktemp + /usr/sbin/sed -e s/NAME_SPACE/pitr-to-new-cluster-17154/g + yq eval '.spec.upgradeOptions.apply="Never"' + local LAST_ERR=/tmp/tmp.BoZJQN9WXt + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.4ftNbTFNBC perconaservermongodb.psmdb.percona.com/some-name-target created + cat /tmp/tmp.BoZJQN9WXt + rm /tmp/tmp.4ftNbTFNBC /tmp/tmp.BoZJQN9WXt + return 0 + wait_for_running some-name-target-rs0 3 + local name=some-name-target-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name-target ++ seq 0 2 + for i in $(seq 0 $last_pod) + [[ 0 -eq 2 ]] + wait_pod some-name-target-rs0-0 + local pod=some-name-target-rs0-0 + set +o xtrace waiting for pod/some-name-target-rs0-0 to be ready..........OK + for i in $(seq 0 $last_pod) + [[ 1 -eq 2 ]] + wait_pod some-name-target-rs0-1 + local pod=some-name-target-rs0-1 + set +o xtrace waiting for pod/some-name-target-rs0-1 to be ready.....OK + for i in $(seq 0 $last_pod) + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ye7DDLoawF +++ mktemp ++ local LAST_ERR=/tmp/tmp.Bg5MBXri6a ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ye7DDLoawF ++ cat /tmp/tmp.Bg5MBXri6a ++ rm /tmp/tmp.ye7DDLoawF /tmp/tmp.Bg5MBXri6a ++ return 0 + [[ '' == true ]] + wait_pod some-name-target-rs0-2 + local pod=some-name-target-rs0-2 + set +o xtrace waiting for pod/some-name-target-rs0-2 to be ready.......OK ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BoDhkmTJO6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.bFex6wiM4d ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].nonvoting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.BoDhkmTJO6 ++ cat /tmp/tmp.bFex6wiM4d ++ rm /tmp/tmp.BoDhkmTJO6 /tmp/tmp.bFex6wiM4d ++ return 0 + [[ '' == true ]] ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7hSY5BCu0t +++ mktemp ++ local LAST_ERR=/tmp/tmp.nGn7NXWhlu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.7hSY5BCu0t ++ cat /tmp/tmp.nGn7NXWhlu ++ rm /tmp/tmp.7hSY5BCu0t /tmp/tmp.nGn7NXWhlu ++ return 0 + [[ '' == true ]] + sleep 10 + [[ true == true ]] + set +x Waiting for cluster readyness................ + wait_backup_agent some-name-target-rs0-0 + local agent_pod=some-name-target-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-0...2026-06-09T09:42:40.000+0000 I listening for the commands + wait_backup_agent some-name-target-rs0-1 + local agent_pod=some-name-target-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-1...2026-06-09T09:43:17.000+0000 I listening for the commands + wait_backup_agent some-name-target-rs0-2 + local agent_pod=some-name-target-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-target-rs0-2...2026-06-09T09:43:20.000+0000 I listening for the commands + wait_for_pbm_operations some-name-target + local cluster=some-name-target + set +o xtrace waiting for PBM operation to finish.OK + desc 'Case 1: Logical restore' + set +o xtrace ----------------------------------------------------------------------------------- Case 1: Logical restore ----------------------------------------------------------------------------------- + test_case_1 + run_restore some-name-target backup-minio logical + local cluster=some-name-target + local backup_name=backup-minio + local backup_type=logical ++ get_backup_dest backup-minio-logical ++ local backup_name=backup-minio-logical ++ kubectl_bin get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.QGB8qkKA44 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iBb8LnmvtZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-logical -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QGB8qkKA44 ++ cat /tmp/tmp.iBb8LnmvtZ ++ rm /tmp/tmp.QGB8qkKA44 /tmp/tmp.iBb8LnmvtZ ++ return 0 + local destination=operator-testing/some-name-source/2026-06-09T09:38:26Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-logical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/' + /usr/sbin/sed -e s/pitrType:/type:/ + /usr/sbin/sed -e 's/type:/type: logical/' + /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2026-06-09T09:38:26Z|' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.R7rGyP259K ++ mktemp + local LAST_ERR=/tmp/tmp.nVidvs48wN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.R7rGyP259K perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-logical created + cat /tmp/tmp.nVidvs48wN + rm /tmp/tmp.R7rGyP259K /tmp/tmp.nVidvs48wN + return 0 + wait_restore backup-minio-logical some-name-target requested 0 + local backup_name=backup-minio-logical + local cluster_name=some-name-target + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK Waiting psmdb-restore/restore-backup-minio-logical to reach state "requested" .OK after 0 minutes + [[ 0 -eq 1 ]] + compare_pbm_config some-name-target source + local cluster=some-name-target + local compare_suffix=source + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.oAauGnMJjD ++ mktemp + local LAST_ERR=/tmp/tmp.VTjkrjgHkI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.oAauGnMJjD + cat /tmp/tmp.VTjkrjgHkI + rm /tmp/tmp.oAauGnMJjD /tmp/tmp.VTjkrjgHkI + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/pbm_config_source.yml /tmp/tmp.KlgUW3jDu0/pbm_config.yml + log 'Backup source is set as main storage during restore: OK' + set +o xtrace [2026-06-09T09:43:36+0000] Backup source is set as main storage during restore: OK ++ date +%s + local now_ts=1780998216 + wait_restore backup-minio-logical some-name-target ready + local backup_name=backup-minio-logical + local cluster_name=some-name-target + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-logical object to be created.OK Waiting psmdb-restore/restore-backup-minio-logical to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name-target + local cluster_name=some-name-target + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ILaJCJLiF3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.obfgVN62JT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ILaJCJLiF3 ++ cat /tmp/tmp.obfgVN62JT ++ rm /tmp/tmp.ILaJCJLiF3 /tmp/tmp.obfgVN62JT ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_pbm_config some-name-target target + local cluster=some-name-target + local compare_suffix=target + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.jpI6HbUvSL ++ mktemp + local LAST_ERR=/tmp/tmp.8zUVnHjPhg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.jpI6HbUvSL + cat /tmp/tmp.8zUVnHjPhg + rm /tmp/tmp.jpI6HbUvSL /tmp/tmp.8zUVnHjPhg + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/pbm_config_target.yml /tmp/tmp.KlgUW3jDu0/pbm_config.yml + log 'Main storage is reverted after backup-minio-logical: OK' + set +o xtrace [2026-06-09T09:44:00+0000] Main storage is reverted after backup-minio-logical: OK + wait_for_pbm_operations some-name-target + local cluster=some-name-target + set +o xtrace waiting for PBM operation to finish.OK + check_if_resync_triggered some-name-target 1780998216 + local cluster=some-name-target + local ts=1780998216 + log 'checking if resync is triggered after 1780998216' + set +o xtrace [2026-06-09T09:44:02+0000] checking if resync is triggered after 1780998216 ++ run_mongo 'use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})' backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 ++ local 'command=use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})' ++ local uri=backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local replica_set=rs0 ++ [[ backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wv9pr1JdNz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZhcR61c243 +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.wv9pr1JdNz +++ cat /tmp/tmp.ZhcR61c243 +++ rm /tmp/tmp.wv9pr1JdNz /tmp/tmp.ZhcR61c243 +++ return 0 ++ local client_container=psmdb-client-bb8b97679-77qzg ++ kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xZ6x34glSF +++ mktemp ++ local LAST_ERR=/tmp/tmp.8e9fYpziXY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.xZ6x34glSF ++ cat /tmp/tmp.8e9fYpziXY command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.xZ6x34glSF ++ cat /tmp/tmp.8e9fYpziXY command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998216 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.xZ6x34glSF ++ cat /tmp/tmp.8e9fYpziXY command terminated with exit code 1 ++ sleep 8 ++ cat /tmp/tmp.xZ6x34glSF ++ cat /tmp/tmp.8e9fYpziXY command terminated with exit code 1 ++ rm /tmp/tmp.xZ6x34glSF /tmp/tmp.8e9fYpziXY ++ return 1 + local $'count=Percona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("c436dd4a-0150-438e-86ed-e9bff977144c") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("9975bd71-de11-4a84-ab2b-447404ee2b7e") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye' + [[ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c436dd4a-0150-438e-86ed-e9bff977144c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("9975bd71-de11-4a84-ab2b-447404ee2b7e") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye -lt 1 ]] run: line 91: [[: Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c436dd4a-0150-438e-86ed-e9bff977144c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("9975bd71-de11-4a84-ab2b-447404ee2b7e") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye: arithmetic syntax error: invalid arithmetic operator (error token is ".4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("c436dd4a-0150-438e-86ed-e9bff977144c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("9975bd71-de11-4a84-ab2b-447404ee2b7e") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("2b4a9f1d-6626-4ead-9f90-ada90985bcdc") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye") + compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-06-09T09:44:20+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Sy7KFsJF4o +++ mktemp ++ local LAST_ERR=/tmp/tmp.0ON6VgACh2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Sy7KFsJF4o ++ cat /tmp/tmp.0ON6VgACh2 ++ rm /tmp/tmp.Sy7KFsJF4o /tmp/tmp.0ON6VgACh2 ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Sqkc2ndikJ ++ mktemp + local LAST_ERR=/tmp/tmp.T8r57PMdWe + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Sqkc2ndikJ + cat /tmp/tmp.T8r57PMdWe + rm /tmp/tmp.Sqkc2ndikJ /tmp/tmp.T8r57PMdWe + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.KlgUW3jDu0/find-2nd + log 'Data is restored from backup-minio-logical: OK' + set +o xtrace [2026-06-09T09:44:22+0000] Data is restored from backup-minio-logical: OK + desc 'Case 1: Logical restore: OK' + set +o xtrace ----------------------------------------------------------------------------------- Case 1: Logical restore: OK ----------------------------------------------------------------------------------- + desc 'Case 2: Physical restore' + set +o xtrace ----------------------------------------------------------------------------------- Case 2: Physical restore ----------------------------------------------------------------------------------- + test_case_2 + run_restore some-name-target backup-minio physical + local cluster=some-name-target + local backup_name=backup-minio + local backup_type=physical ++ get_backup_dest backup-minio-physical ++ local backup_name=backup-minio-physical ++ kubectl_bin get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' +++ mktemp ++ sed 's|azure://||' ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.fEt6RdF08X +++ mktemp ++ local LAST_ERR=/tmp/tmp.PxFWc1OKkz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup backup-minio-physical -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fEt6RdF08X ++ cat /tmp/tmp.PxFWc1OKkz ++ rm /tmp/tmp.fEt6RdF08X /tmp/tmp.PxFWc1OKkz ++ return 0 + local destination=operator-testing/some-name-source/2026-06-09T09:39:51Z + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/restore-backupsource.yml + /usr/sbin/sed -e 's/name:/name: restore-backup-minio-physical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/' + /usr/sbin/sed -e 's/type:/type: physical/' + /usr/sbin/sed -e s/pitrType:/type:/ + /usr/sbin/sed -e 's|DESTINATION|operator-testing/some-name-source/2026-06-09T09:39:51Z|' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HhpSAmad7B ++ mktemp + local LAST_ERR=/tmp/tmp.CKtdNQDFAG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HhpSAmad7B perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-physical created + cat /tmp/tmp.CKtdNQDFAG + rm /tmp/tmp.HhpSAmad7B /tmp/tmp.CKtdNQDFAG + return 0 + wait_restore backup-minio-physical some-name-target requested 0 + local backup_name=backup-minio-physical + local cluster_name=some-name-target + local target_state=requested + local wait_cluster_consistency=0 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-physical object to be created.OK Waiting psmdb-restore/restore-backup-minio-physical to reach state "requested" ..OK after 1 minutes + [[ 0 -eq 1 ]] + compare_pbm_config some-name-target source mongod + local cluster=some-name-target + local compare_suffix=source + local container=mongod + local pbm_binary=pbm + [[ mongod == mongod ]] + pbm_binary=/opt/percona/pbm + kubectl_bin exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.qc1bB9oJfP ++ mktemp + local LAST_ERR=/tmp/tmp.EVBBp4ck0e + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c mongod -- /opt/percona/pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.qc1bB9oJfP + cat /tmp/tmp.EVBBp4ck0e + rm /tmp/tmp.qc1bB9oJfP /tmp/tmp.EVBBp4ck0e + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/pbm_config_source.yml /tmp/tmp.KlgUW3jDu0/pbm_config.yml + log 'Backup source is set as main storage during restore: OK' + set +o xtrace [2026-06-09T09:46:21+0000] Backup source is set as main storage during restore: OK ++ date +%s + local now_ts=1780998381 + wait_restore backup-minio-physical some-name-target ready + local backup_name=backup-minio-physical + local cluster_name=some-name-target + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-backup-minio-physical object to be created.OK Waiting psmdb-restore/restore-backup-minio-physical to reach state "ready" ..OK after 1 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name-target + local cluster_name=some-name-target + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VcDO4AaisP +++ mktemp ++ local LAST_ERR=/tmp/tmp.pjfyxSbt7I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VcDO4AaisP ++ cat /tmp/tmp.pjfyxSbt7I ++ rm /tmp/tmp.VcDO4AaisP /tmp/tmp.pjfyxSbt7I ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 1 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L7xPdIcS0A +++ mktemp ++ local LAST_ERR=/tmp/tmp.OnCap1KdB5 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.L7xPdIcS0A ++ cat /tmp/tmp.OnCap1KdB5 ++ rm /tmp/tmp.L7xPdIcS0A /tmp/tmp.OnCap1KdB5 ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 2 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fPZu7DHLFL +++ mktemp ++ local LAST_ERR=/tmp/tmp.dgo6am9mv0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.fPZu7DHLFL ++ cat /tmp/tmp.dgo6am9mv0 ++ rm /tmp/tmp.fPZu7DHLFL /tmp/tmp.dgo6am9mv0 ++ return 0 + [[ error == ready ]] + let retry+=1 + '[' 3 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.esVVjDxp0T +++ mktemp ++ local LAST_ERR=/tmp/tmp.SqehK0TqKA ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.esVVjDxp0T ++ cat /tmp/tmp.SqehK0TqKA ++ rm /tmp/tmp.esVVjDxp0T /tmp/tmp.SqehK0TqKA ++ return 0 + [[ error == ready ]] + let retry+=1 + '[' 4 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4mfQ3Bkj5e +++ mktemp ++ local LAST_ERR=/tmp/tmp.vunJ1zTvY6 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4mfQ3Bkj5e ++ cat /tmp/tmp.vunJ1zTvY6 ++ rm /tmp/tmp.4mfQ3Bkj5e /tmp/tmp.vunJ1zTvY6 ++ return 0 + [[ error == ready ]] + let retry+=1 + '[' 5 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.S1HAsVp8z5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.gp97AztpaY ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.S1HAsVp8z5 ++ cat /tmp/tmp.gp97AztpaY ++ rm /tmp/tmp.S1HAsVp8z5 /tmp/tmp.gp97AztpaY ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 6 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.B7F5axeLSn +++ mktemp ++ local LAST_ERR=/tmp/tmp.QkPFhzRRvL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.B7F5axeLSn ++ cat /tmp/tmp.QkPFhzRRvL ++ rm /tmp/tmp.B7F5axeLSn /tmp/tmp.QkPFhzRRvL ++ return 0 + [[ initializing == ready ]] + let retry+=1 + '[' 7 -ge 32 ']' + echo -n . .+ sleep 10 ++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yUHAQQGMJ7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xYHCY56GhU ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yUHAQQGMJ7 ++ cat /tmp/tmp.xYHCY56GhU ++ rm /tmp/tmp.yUHAQQGMJ7 /tmp/tmp.xYHCY56GhU ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_pbm_config some-name-target target + local cluster=some-name-target + local compare_suffix=target + local container=backup-agent + local pbm_binary=pbm + [[ backup-agent == mongod ]] + kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm config ++ mktemp + local LAST_OUT=/tmp/tmp.vLTKxb0qQk ++ mktemp + local LAST_ERR=/tmp/tmp.NwahWd3GXL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm config + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.vLTKxb0qQk + cat /tmp/tmp.NwahWd3GXL + rm /tmp/tmp.vLTKxb0qQk /tmp/tmp.NwahWd3GXL + return 0 + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/pbm_config_target.yml /tmp/tmp.KlgUW3jDu0/pbm_config.yml + log 'Main storage is reverted after backup-minio-physical: OK' + set +o xtrace [2026-06-09T09:49:29+0000] Main storage is reverted after backup-minio-physical: OK + wait_for_pbm_operations some-name-target + local cluster=some-name-target + set +o xtrace waiting for PBM operation to finish.OK + check_if_resync_triggered some-name-target 1780998381 + local cluster=some-name-target + local ts=1780998381 + log 'checking if resync is triggered after 1780998381' + set +o xtrace [2026-06-09T09:49:30+0000] checking if resync is triggered after 1780998381 ++ run_mongo 'use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})' backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 ++ local 'command=use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})' ++ local uri=backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 ++ local driver=mongodb+srv ++ local suffix=.svc.cluster.local ++ local mongo_flag= ++ local replica_set=rs0 ++ [[ backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] +++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uNQux5WpoP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jJuGgKTV6k +++ local exit_status=0 +++ local timeout=4 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 -a -n 1 ']' +++ break +++ cat /tmp/tmp.uNQux5WpoP +++ cat /tmp/tmp.jJuGgKTV6k +++ rm /tmp/tmp.uNQux5WpoP /tmp/tmp.jJuGgKTV6k +++ return 0 ++ local client_container=psmdb-client-bb8b97679-77qzg ++ kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EhXIWSyoka +++ mktemp ++ local LAST_ERR=/tmp/tmp.tvoe11wXa0 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.EhXIWSyoka ++ cat /tmp/tmp.tvoe11wXa0 command terminated with exit code 1 ++ sleep 0 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.EhXIWSyoka ++ cat /tmp/tmp.tvoe11wXa0 command terminated with exit code 1 ++ sleep 4 ++ for i in $(seq 0 2) ++ set +e ++ kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use admin\n db.pbmCmd.countDocuments({cmd: '\''resync'\'', ts: { '\''$gt'\'': 1780998381 }})\n'\'' | mongo mongodb+srv://backup:backup123456@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.EhXIWSyoka ++ cat /tmp/tmp.tvoe11wXa0 command terminated with exit code 1 ++ sleep 8 ++ cat /tmp/tmp.EhXIWSyoka ++ cat /tmp/tmp.tvoe11wXa0 command terminated with exit code 1 ++ rm /tmp/tmp.EhXIWSyoka /tmp/tmp.tvoe11wXa0 ++ return 1 + local $'count=Percona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("8fff52b4-ee06-4b8e-8f1e-11979ec903fb") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("b9e127a2-7d0c-4c07-842c-7390ef402919") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye\nPercona Server for MongoDB shell version v4.4.29-28\nconnecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false\nImplicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") }\nPercona Server for MongoDB server version: v8.0.23-10\nWARNING: shell and server versions do not match\nswitched to db admin\nuncaught exception: SyntaxError: expected property name, got \':\' :\n@(shell):1:45\nbye' + [[ Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8fff52b4-ee06-4b8e-8f1e-11979ec903fb") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b9e127a2-7d0c-4c07-842c-7390ef402919") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye -lt 1 ]] run: line 91: [[: Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8fff52b4-ee06-4b8e-8f1e-11979ec903fb") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b9e127a2-7d0c-4c07-842c-7390ef402919") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye: arithmetic syntax error: invalid arithmetic operator (error token is ".4.29-28 connecting to: mongodb://some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("8fff52b4-ee06-4b8e-8f1e-11979ec903fb") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("b9e127a2-7d0c-4c07-842c-7390ef402919") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("6c6650ca-29b0-44e6-97cb-16851a8f2a1f") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db admin uncaught exception: SyntaxError: expected property name, got ':' : @(shell):1:45 bye") + compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 -2nd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local postfix=-2nd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-06-09T09:49:48+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V7XqYRnNIu +++ mktemp ++ local LAST_ERR=/tmp/tmp.jpPgBYvvVs ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V7XqYRnNIu ++ cat /tmp/tmp.jpPgBYvvVs ++ rm /tmp/tmp.V7XqYRnNIu /tmp/tmp.jpPgBYvvVs ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.Ge0k8SzUoJ ++ mktemp + local LAST_ERR=/tmp/tmp.VCwqSJdInz + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Ge0k8SzUoJ + cat /tmp/tmp.VCwqSJdInz + rm /tmp/tmp.Ge0k8SzUoJ /tmp/tmp.VCwqSJdInz + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find-2nd.json /tmp/tmp.KlgUW3jDu0/find-2nd + log 'Data is restored from backup-minio-physical: OK' + set +o xtrace [2026-06-09T09:49:51+0000] Data is restored from backup-minio-physical: OK + desc 'Case 2: Physical restore: OK' + set +o xtrace ----------------------------------------------------------------------------------- Case 2: Physical restore: OK ----------------------------------------------------------------------------------- + desc 'Case 3: Backup and restore on some-name-target' + set +o xtrace ----------------------------------------------------------------------------------- Case 3: Backup and restore on some-name-target ----------------------------------------------------------------------------------- + test_case_3 + local backup_name=target-backup-minio + write_document some-name-target 100504 + local cluster=some-name-target + local x=100504 + log 'write document: 100504' + set +o xtrace [2026-06-09T09:49:51+0000] write document: 100504 + run_mongo 'use myApp\n db.test.insert({ x: 100504 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100504 })' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QhmDVyeTt6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Slpp5d4WSp ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QhmDVyeTt6 ++ cat /tmp/tmp.Slpp5d4WSp ++ rm /tmp/tmp.QhmDVyeTt6 /tmp/tmp.Slpp5d4WSp ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.zycWPuzJz9 ++ mktemp + local LAST_ERR=/tmp/tmp.4SxDDbdiUf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100504 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.zycWPuzJz9 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("02b3581a-84b1-4bed-8d4c-1e70ee634230") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.4SxDDbdiUf + rm /tmp/tmp.zycWPuzJz9 /tmp/tmp.4SxDDbdiUf + return 0 + run_backup some-name-target target-backup-minio logical + local cluster=some-name-target + local name=target-backup-minio + local type=logical + desc 'run backup target-backup-minio-logical' + set +o xtrace ----------------------------------------------------------------------------------- run backup target-backup-minio-logical ----------------------------------------------------------------------------------- + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/backup.yml + /usr/sbin/sed -e 's/name:/name: target-backup-minio-logical/' + /usr/sbin/sed -e 's/clusterName:/clusterName: some-name-target/' + /usr/sbin/sed -e 's/type:/type: logical/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tv8OAjXFYp ++ mktemp + local LAST_ERR=/tmp/tmp.hg0OhM7rEN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.tv8OAjXFYp perconaservermongodbbackup.psmdb.percona.com/target-backup-minio-logical created + cat /tmp/tmp.hg0OhM7rEN + rm /tmp/tmp.tv8OAjXFYp /tmp/tmp.hg0OhM7rEN + return 0 + wait_backup target-backup-minio-logical + local backup_name=target-backup-minio-logical + local target_state=ready + set +o xtrace waiting for target-backup-minio-logical to reach ready state.......OK + log 'checking target-backup-minio-logical in storage' + set +o xtrace [2026-06-09T09:50:09+0000] checking target-backup-minio-logical in storage + check_backup_in_storage target-backup-minio-logical minio rs0 myApp.test.gz + local backup=target-backup-minio-logical + local storage_type=minio + local replset=rs0 + local file=myApp.test.gz + local protocol=http + local endpoint ++ get_backup_dest target-backup-minio-logical ++ local backup_name=target-backup-minio-logical ++ /usr/sbin/sed 's|https://engk8soperators.blob.core.windows.net/||' ++ kubectl_bin get psmdb-backup target-backup-minio-logical -o 'jsonpath={.status.destination}' ++ sed -e 's/.json$//' ++ sed 's|s3://||' ++ sed 's|azure://||' +++ mktemp ++ sed 's|gs://||' ++ local LAST_OUT=/tmp/tmp.5iND4UkOhV +++ mktemp ++ local LAST_ERR=/tmp/tmp.XvhI4VQAXL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb-backup target-backup-minio-logical -o 'jsonpath={.status.destination}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.5iND4UkOhV ++ cat /tmp/tmp.XvhI4VQAXL ++ rm /tmp/tmp.5iND4UkOhV /tmp/tmp.XvhI4VQAXL ++ return 0 + backup_dest=operator-testing/some-name-target/2026-06-09T09:49:57Z + case ${storage_type} in + endpoint=minio-service + kubectl_bin run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/some-name-target/2026-06-09T09:49:57Z/rs0/myApp.test.gz + grep myApp.test.gz ++ mktemp + local LAST_OUT=/tmp/tmp.OySvdfUkqe ++ mktemp + local LAST_ERR=/tmp/tmp.6JZwTxFreH + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl run -i --rm aws-cli --image=docker.io/perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --no-verify-ssl --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/some-name-target/2026-06-09T09:49:57Z/rs0/myApp.test.gz + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OySvdfUkqe + cat /tmp/tmp.6JZwTxFreH All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-to-new-cluster-17154 + rm /tmp/tmp.OySvdfUkqe /tmp/tmp.6JZwTxFreH + return 0 2026-06-09 09:50:00 118 myApp.test.gz 2026-06-09 09:50:00 118 myApp.test.gz + log 'target-backup-minio is uploaded to correct storage: OK' + set +o xtrace [2026-06-09T09:50:14+0000] target-backup-minio is uploaded to correct storage: OK + log 'enabling PiTR on some-name-target' + set +o xtrace [2026-06-09T09:50:14+0000] enabling PiTR on some-name-target + kubectl_bin patch psmdb some-name-target --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": true}}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.kykikmydVX ++ mktemp + local LAST_ERR=/tmp/tmp.gWtI2lwDA5 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl patch psmdb some-name-target --type=merge --patch '{"spec": {"backup": {"pitr": {"enabled": true}}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.kykikmydVX perconaservermongodb.psmdb.percona.com/some-name-target patched + cat /tmp/tmp.gWtI2lwDA5 + rm /tmp/tmp.kykikmydVX /tmp/tmp.gWtI2lwDA5 + return 0 + write_document some-name-target 100505 + local cluster=some-name-target + local x=100505 + log 'write document: 100505' + set +o xtrace [2026-06-09T09:50:16+0000] write document: 100505 + run_mongo 'use myApp\n db.test.insert({ x: 100505 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100505 })' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.X4ynF6GGRO +++ mktemp ++ local LAST_ERR=/tmp/tmp.i4PJSxNxGE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.X4ynF6GGRO ++ cat /tmp/tmp.i4PJSxNxGE ++ rm /tmp/tmp.X4ynF6GGRO /tmp/tmp.i4PJSxNxGE ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.EXYWgml7X5 ++ mktemp + local LAST_ERR=/tmp/tmp.2fB8wuswCr + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100505 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EXYWgml7X5 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("882b9d6d-90b4-4acc-b014-1dd95ef28fe9") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.2fB8wuswCr + rm /tmp/tmp.EXYWgml7X5 /tmp/tmp.2fB8wuswCr + return 0 + write_document some-name-target 100506 + local cluster=some-name-target + local x=100506 + log 'write document: 100506' + set +o xtrace [2026-06-09T09:50:18+0000] write document: 100506 + run_mongo 'use myApp\n db.test.insert({ x: 100506 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100506 })' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4TpaAVnBEd +++ mktemp ++ local LAST_ERR=/tmp/tmp.pSxRwFsIZr ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.4TpaAVnBEd ++ cat /tmp/tmp.pSxRwFsIZr ++ rm /tmp/tmp.4TpaAVnBEd /tmp/tmp.pSxRwFsIZr ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.8Em3fTxqdV ++ mktemp + local LAST_ERR=/tmp/tmp.Udx40izghL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100506 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.8Em3fTxqdV Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("917cb1a2-a845-4748-a3a7-3d9b2e5fc5e9") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.Udx40izghL + rm /tmp/tmp.8Em3fTxqdV /tmp/tmp.Udx40izghL + return 0 + write_document some-name-target 100507 + local cluster=some-name-target + local x=100507 + log 'write document: 100507' + set +o xtrace [2026-06-09T09:50:20+0000] write document: 100507 + run_mongo 'use myApp\n db.test.insert({ x: 100507 })' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb + local 'command=use myApp\n db.test.insert({ x: 100507 })' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ymCXy6uKic +++ mktemp ++ local LAST_ERR=/tmp/tmp.hpTrdpoUEH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ymCXy6uKic ++ cat /tmp/tmp.hpTrdpoUEH ++ rm /tmp/tmp.ymCXy6uKic /tmp/tmp.hpTrdpoUEH ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.NCmYwwqDya ++ mktemp + local LAST_ERR=/tmp/tmp.q499VelIoX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100507 })\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.NCmYwwqDya Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("e1f459f2-3c4e-4325-a7dd-29c21ad9bd9c") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.q499VelIoX + rm /tmp/tmp.NCmYwwqDya /tmp/tmp.q499VelIoX + return 0 + sleep_with_log 3 'wait for data to be replicated' + local d=3 + local 'msg=wait for data to be replicated' + log 'wait for data to be replicated. sleeping for 3 seconds' + set +o xtrace [2026-06-09T09:50:23+0000] wait for data to be replicated. sleeping for 3 seconds + sleep 3 + compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 -3rd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local postfix=-3rd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-06-09T09:50:26+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb .svc.cluster.local '' + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yYfUBVsiUq +++ mktemp ++ local LAST_ERR=/tmp/tmp.3b9y2jO1Y2 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yYfUBVsiUq ++ cat /tmp/tmp.3b9y2jO1Y2 ++ rm /tmp/tmp.yYfUBVsiUq /tmp/tmp.3b9y2jO1Y2 ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.GMdaJeNcox ++ mktemp + local LAST_ERR=/tmp/tmp.0rem1qYDEX + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.GMdaJeNcox + cat /tmp/tmp.0rem1qYDEX + rm /tmp/tmp.GMdaJeNcox /tmp/tmp.0rem1qYDEX + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find-3rd.json /tmp/tmp.KlgUW3jDu0/find-3rd + log 'Data is ready for PiTR: OK' + set +o xtrace [2026-06-09T09:50:28+0000] Data is ready for PiTR: OK + sleep_with_log 70 'wait for oplog chunks to be uploaded' + local d=70 + local 'msg=wait for oplog chunks to be uploaded' + log 'wait for oplog chunks to be uploaded. sleeping for 70 seconds' + set +o xtrace [2026-06-09T09:50:28+0000] wait for oplog chunks to be uploaded. sleeping for 70 seconds + sleep 70 +++ get_latest_restorable_time some-name-target-rs0 +++ local cluster=some-name-target-rs0 +++ local first_timestamp +++ local second_timestamp +++ local retry=0 +++ [[ '' != '' ]] ++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Nbb9OemKW2 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.9rz9gi8JDk ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.Nbb9OemKW2 ++++ cat /tmp/tmp.9rz9gi8JDk ++++ rm /tmp/tmp.Nbb9OemKW2 /tmp/tmp.9rz9gi8JDk ++++ return 0 +++ first_timestamp=null +++ sleep 5 +++ [[ null != '' ]] +++ [[ null != null ]] +++ let retry+=1 +++ [[ 1 -gt 30 ]] +++ [[ null != '' ]] +++ [[ null != null ]] ++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.patBmOoTh1 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.KLgSQ3nvOd ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.patBmOoTh1 ++++ cat /tmp/tmp.KLgSQ3nvOd ++++ rm /tmp/tmp.patBmOoTh1 /tmp/tmp.KLgSQ3nvOd ++++ return 0 +++ first_timestamp=1780998702 +++ sleep 5 +++ [[ 1780998702 != '' ]] +++ [[ 1780998702 != null ]] ++++ kubectl_bin exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ jq '.backups.pitrChunks.pitrChunks | last | .range.end' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.rDMmgDJboM +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.vDlP3mU92J ++++ local exit_status=0 ++++ local timeout=4 +++++ seq 0 2 ++++ for i in $(seq 0 2) ++++ set +e ++++ kubectl exec some-name-target-rs0-0 -c backup-agent -- pbm status -o json ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 -a -n 1 ']' ++++ break ++++ cat /tmp/tmp.rDMmgDJboM ++++ cat /tmp/tmp.vDlP3mU92J ++++ rm /tmp/tmp.rDMmgDJboM /tmp/tmp.vDlP3mU92J ++++ return 0 +++ second_timestamp=1780998702 +++ let retry+=1 +++ [[ 2 -gt 30 ]] +++ [[ 1780998702 != '' ]] +++ [[ 1780998702 != null ]] +++ [[ 1780998702 == 1780998702 ]] +++ /usr/sbin/date -u -d @1780998702 +%Y-%m-%dT%H:%M:%SZ ++ format_pitr_target 2026-06-09T09:51:42Z ++ local target=2026-06-09T09:51:42Z ++ echo 2026-06-09T09:51:42Z ++ sed 's/T/ /' ++ sed s/Z// + local 'pitrTarget=2026-06-09 09:51:42' + log 'dropping collection: myApp.test' + set +o xtrace [2026-06-09T09:51:53+0000] dropping collection: myApp.test + run_mongo 'use myApp\n db.test.drop()' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local 'command=use myApp\n db.test.drop()' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb+srv + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MfJuXicQTJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.ynFu4s7P8E ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MfJuXicQTJ ++ cat /tmp/tmp.ynFu4s7P8E ++ rm /tmp/tmp.MfJuXicQTJ /tmp/tmp.ynFu4s7P8E ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.RZ0VOe0Fdl ++ mktemp + local LAST_ERR=/tmp/tmp.r0EwHPq1tk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RZ0VOe0Fdl Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-target-rs0-1.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-2.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017,some-name-target-rs0-0.some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false Implicit session: session { "id" : UUID("4db5e0d2-16b2-4c65-b83e-8a3793cf89fa") } Percona Server for MongoDB server version: v8.0.23-10 WARNING: shell and server versions do not match switched to db myApp true bye + cat /tmp/tmp.r0EwHPq1tk + rm /tmp/tmp.RZ0VOe0Fdl /tmp/tmp.r0EwHPq1tk + return 0 + log 'PiTR target is 2026-06-09 09:51:42' + set +o xtrace [2026-06-09T09:51:56+0000] PiTR target is 2026-06-09 09:51:42 + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/conf/restore-target.yml + /usr/sbin/sed -e 's/date:/date: 2026-06-09 09:51:42/' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.KDtPBtW6i7 ++ mktemp + local LAST_ERR=/tmp/tmp.DwfAseaLYR + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.KDtPBtW6i7 perconaservermongodbrestore.psmdb.percona.com/restore-target-backup-minio-logical created + cat /tmp/tmp.DwfAseaLYR + rm /tmp/tmp.KDtPBtW6i7 /tmp/tmp.DwfAseaLYR + return 0 + wait_restore target-backup-minio-logical some-name-target ready + local backup_name=target-backup-minio-logical + local cluster_name=some-name-target + local target_state=ready + local wait_cluster_consistency=1 + local wait_time=1780 + local ok_if_ready=0 + set +o xtrace Waiting for the psmdb-restore/restore-target-backup-minio-logical object to be created.OK Waiting psmdb-restore/restore-target-backup-minio-logical to reach state "ready" .OK after 0 minutes + [[ 1 -eq 1 ]] + wait_cluster_consistency some-name-target + local cluster_name=some-name-target + local wait_time=32 + retry=0 + sleep 7 + echo -n 'waiting for cluster readyness' waiting for cluster readyness++ kubectl_bin get psmdb some-name-target -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QHfOabPtz3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.QuJH4KfFd3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get psmdb some-name-target -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.QHfOabPtz3 ++ cat /tmp/tmp.QuJH4KfFd3 ++ rm /tmp/tmp.QHfOabPtz3 /tmp/tmp.QuJH4KfFd3 ++ return 0 + [[ ready == ready ]] + echo .OK .OK + compare_mongo_cmd find myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 -3rd .svc.cluster.local myApp test 'sort( { x: 1 } )' + local command=find + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local postfix=-3rd + local suffix=.svc.cluster.local + local database=myApp + local collection=test + local 'sort=sort( { x: 1 } )' + local tls=false + local replicaset= + local 'full_command=db.test.find()' + [[ -n sort( { x: 1 } ) ]] + full_command='db.test.find().sort( { x: 1 } )' + log 'running db.test.find().sort( { x: 1 } ) in myApp' + set +o xtrace [2026-06-09T09:52:39+0000] running db.test.find().sort( { x: 1 } ) in myApp + [[ false == true ]] + mongo_command=run_mongo + run_mongo 'use myApp\n db.test.find().sort( { x: 1 } )' myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 mongodb .svc.cluster.local '' + local 'command=use myApp\n db.test.find().sort( { x: 1 } )' + local uri=myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local replica_set=rs0 + [[ myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154 == *cfg* ]] + grep -E -v 'I NETWORK|W NETWORK|F NETWORK|"c":"NETWORK"|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' + /usr/sbin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V4E88wVWAW +++ mktemp ++ local LAST_ERR=/tmp/tmp.ot9pyK9sXZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.V4E88wVWAW ++ cat /tmp/tmp.ot9pyK9sXZ ++ rm /tmp/tmp.V4E88wVWAW /tmp/tmp.ot9pyK9sXZ ++ return 0 + local client_container=psmdb-client-bb8b97679-77qzg + kubectl_bin exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' ++ mktemp + local LAST_OUT=/tmp/tmp.78R7hLOGi8 ++ mktemp + local LAST_ERR=/tmp/tmp.uthH1k6Heb + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl exec psmdb-client-bb8b97679-77qzg -- bash -c 'printf '\''use myApp\n db.test.find().sort( { x: 1 } )\n'\'' | mongo mongodb://myApp:myPass@some-name-target-rs0.pitr-to-new-cluster-17154.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.78R7hLOGi8 + cat /tmp/tmp.uthH1k6Heb + rm /tmp/tmp.78R7hLOGi8 /tmp/tmp.uthH1k6Heb + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2058/e2e-tests/pitr-to-new-cluster/compare/find-3rd.json /tmp/tmp.KlgUW3jDu0/find-3rd + log 'Data is restored from target-backup-minio-logical: OK' + set +o xtrace [2026-06-09T09:52:42+0000] Data is restored from target-backup-minio-logical: OK + desc 'Case 3: Backup and restore on some-name-target: OK' + set +o xtrace ----------------------------------------------------------------------------------- Case 3: Backup and restore on some-name-target: OK ----------------------------------------------------------------------------------- + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + destroy pitr-to-new-cluster-17154 + local namespace=pitr-to-new-cluster-17154 + local ignore_logs=true + [[ 1 == 1 ]] + echo 'SKIP_DELETE=1, not destroying pitr-to-new-cluster-17154' SKIP_DELETE=1, not destroying pitr-to-new-cluster-17154 + return