Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/logs/pitr-physical.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ main
+ create_infra pitr-physical-18132
+ local ns=pitr-physical-18132
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.TySqePFv4e
++ mktemp
+ local LAST_ERR=/tmp/tmp.YdlqBsw0JE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TySqePFv4e
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.YdlqBsw0JE
+ rm /tmp/tmp.TySqePFv4e /tmp/tmp.YdlqBsw0JE
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.oWbbjR901Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.fH0zXIpkjU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oWbbjR901Q
+ cat /tmp/tmp.fH0zXIpkjU
+ rm /tmp/tmp.oWbbjR901Q /tmp/tmp.fH0zXIpkjU
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.f0nRF3KVtZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.36esyuwma9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.f0nRF3KVtZ
+ cat /tmp/tmp.36esyuwma9
+ rm /tmp/tmp.f0nRF3KVtZ /tmp/tmp.36esyuwma9
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.dL4cfHRYEG
++ mktemp
+ local LAST_ERR=/tmp/tmp.vdXWdOt4FO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dL4cfHRYEG
+ cat /tmp/tmp.vdXWdOt4FO
+ rm /tmp/tmp.dL4cfHRYEG /tmp/tmp.vdXWdOt4FO
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.nz4FwyuUEw
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Ea6ma6dTW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nz4FwyuUEw
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.2Ea6ma6dTW
+ rm /tmp/tmp.nz4FwyuUEw /tmp/tmp.2Ea6ma6dTW
+ return 0
+ check_crd_for_deletion PR-1556-43640f06
+ local git_tag=PR-1556-43640f06
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1556-43640f06/deploy/crd.yaml
++ /usr/bin/sed s/---//g
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dhflr7C0Lz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KUGCM3sAY8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dhflr7C0Lz
++ cat /tmp/tmp.KUGCM3sAY8
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dhflr7C0Lz
++ cat /tmp/tmp.KUGCM3sAY8
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.dhflr7C0Lz
++ cat /tmp/tmp.KUGCM3sAY8
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.dhflr7C0Lz
++ cat /tmp/tmp.KUGCM3sAY8
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.dhflr7C0Lz /tmp/tmp.KUGCM3sAY8
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ xargs kubectl delete ns


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

++ mktemp
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ local LAST_OUT=/tmp/tmp.ssU57dph7g
++ mktemp
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.aKbAehoKju
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_ERR=/tmp/tmp.cxKvXPThjY
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.hthZ4SlVC7
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ssU57dph7g
+ cat /tmp/tmp.cxKvXPThjY
+ rm /tmp/tmp.ssU57dph7g /tmp/tmp.cxKvXPThjY
+ return 0
namespace "pitr-physical-24881" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aKbAehoKju
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.hthZ4SlVC7
+ rm /tmp/tmp.aKbAehoKju /tmp/tmp.hthZ4SlVC7
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.UzdbLoFzqh
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ict3F68yrU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UzdbLoFzqh
+ cat /tmp/tmp.Ict3F68yrU
+ rm /tmp/tmp.UzdbLoFzqh /tmp/tmp.Ict3F68yrU
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.1s5WlDIO8P
++ mktemp
+ local LAST_ERR=/tmp/tmp.daz0dhkkT0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1s5WlDIO8P
namespace/psmdb-operator created
+ cat /tmp/tmp.daz0dhkkT0
+ rm /tmp/tmp.1s5WlDIO8P /tmp/tmp.daz0dhkkT0
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Je9aIjYDcA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZHn8BFJ2kk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Je9aIjYDcA
++ cat /tmp/tmp.ZHn8BFJ2kk
++ rm /tmp/tmp.Je9aIjYDcA /tmp/tmp.ZHn8BFJ2kk
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.gedJHqV80s
++ mktemp
+ local LAST_ERR=/tmp/tmp.E84EkXKLIo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gedJHqV80s
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6" modified.
+ cat /tmp/tmp.E84EkXKLIo
+ rm /tmp/tmp.gedJHqV80s /tmp/tmp.E84EkXKLIo
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.j6K1cX7ngp
++ mktemp
+ local LAST_ERR=/tmp/tmp.JwCypSfpGM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j6K1cX7ngp
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.JwCypSfpGM
+ rm /tmp/tmp.j6K1cX7ngp /tmp/tmp.JwCypSfpGM
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ local LAST_OUT=/tmp/tmp.a3Jq8MHwXb
++ mktemp
+ local LAST_ERR=/tmp/tmp.4nus6cMbh5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.a3Jq8MHwXb
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.4nus6cMbh5
+ rm /tmp/tmp.a3Jq8MHwXb /tmp/tmp.4nus6cMbh5
+ return 0
+ kubectl_bin apply -f -
++ mktemp
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1556-43640f06") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-operator.yaml
+ local LAST_OUT=/tmp/tmp.IiF43UbxK3
++ mktemp
+ local LAST_ERR=/tmp/tmp.pSjP3iOXEE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IiF43UbxK3
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.pSjP3iOXEE
+ rm /tmp/tmp.IiF43UbxK3 /tmp/tmp.pSjP3iOXEE
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WuJDiYW6vw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nRZbp5sGvn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WuJDiYW6vw
++ cat /tmp/tmp.nRZbp5sGvn
++ rm /tmp/tmp.WuJDiYW6vw /tmp/tmp.nRZbp5sGvn
++ return 0
+ wait_pod percona-server-mongodb-operator-5b96847cc9-p2mmf
+ local pod=percona-server-mongodb-operator-5b96847cc9-p2mmf
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-5b96847cc9-p2mmf to be ready.OK
+ create_namespace pitr-physical-18132
+ local namespace=pitr-physical-18132
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-physical-18132'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-physical-18132
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-physical-18132 --ignore-not-found
+ xargs kubectl delete ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.fFBghbGXCI
++ mktemp
+ local LAST_ERR=/tmp/tmp.GWpN4dg4vY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.nqvj1pWhnq
++ mktemp
+ local LAST_ERR=/tmp/tmp.RS4EZvPVfq
+ local exit_status=0
+ local timeout=4
+ awk '{print$1}'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-physical-18132 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fFBghbGXCI
+ cat /tmp/tmp.GWpN4dg4vY
+ rm /tmp/tmp.fFBghbGXCI /tmp/tmp.GWpN4dg4vY
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nqvj1pWhnq
+ cat /tmp/tmp.RS4EZvPVfq
+ rm /tmp/tmp.nqvj1pWhnq /tmp/tmp.RS4EZvPVfq
+ return 0
+ kubectl_bin wait --for=delete namespace pitr-physical-18132
++ mktemp
+ local LAST_OUT=/tmp/tmp.j0DROhWyla
++ mktemp
+ local LAST_ERR=/tmp/tmp.gfUVNH1wJJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace pitr-physical-18132
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.j0DROhWyla
+ cat /tmp/tmp.gfUVNH1wJJ
+ rm /tmp/tmp.j0DROhWyla /tmp/tmp.gfUVNH1wJJ
+ return 0
+ desc 'create namespace pitr-physical-18132'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-physical-18132
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-physical-18132
++ mktemp
+ local LAST_OUT=/tmp/tmp.XowNhuR6bp
++ mktemp
+ local LAST_ERR=/tmp/tmp.P3EuXV7EHC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pitr-physical-18132
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XowNhuR6bp
namespace/pitr-physical-18132 created
+ cat /tmp/tmp.P3EuXV7EHC
+ rm /tmp/tmp.XowNhuR6bp /tmp/tmp.P3EuXV7EHC
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eso1xrZlUN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UxosvjkulA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eso1xrZlUN
++ cat /tmp/tmp.UxosvjkulA
++ rm /tmp/tmp.eso1xrZlUN /tmp/tmp.UxosvjkulA
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6 --namespace=pitr-physical-18132
++ mktemp
+ local LAST_OUT=/tmp/tmp.bUVBQoZxun
++ mktemp
+ local LAST_ERR=/tmp/tmp.aVinZU9MTo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6 --namespace=pitr-physical-18132
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bUVBQoZxun
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1556-43640f06-2-cluster6" modified.
+ cat /tmp/tmp.aVinZU9MTo
+ rm /tmp/tmp.bUVBQoZxun /tmp/tmp.aVinZU9MTo
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Fri May 24 16:36:38 2024
NAMESPACE: pitr-physical-18132
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-physical-18132.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-physical-18132 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-physical-18132

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-physical-18132 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-physical-18132 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uYZXhy7GRI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Vl1ebtPRD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uYZXhy7GRI
++ cat /tmp/tmp.5Vl1ebtPRD
++ rm /tmp/tmp.uYZXhy7GRI /tmp/tmp.5Vl1ebtPRD
++ return 0
+ MINIO_POD=minio-service-57dd49b-hbx6x
+ wait_pod minio-service-57dd49b-hbx6x
+ local pod=minio-service-57dd49b-hbx6x
+ set +o xtrace
waiting for pod/minio-service-57dd49b-hbx6x to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-18132.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.nQc4r0tVru
++ mktemp
+ local LAST_ERR=/tmp/tmp.zoKAM6SdGA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.pitr-physical-18132.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nQc4r0tVru
service/minio-service created
+ cat /tmp/tmp.zoKAM6SdGA
+ rm /tmp/tmp.nQc4r0tVru /tmp/tmp.zoKAM6SdGA
+ return 0
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VcRNbnxnyK
++ mktemp
+ local LAST_ERR=/tmp/tmp.q4KFhCaSxY
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VcRNbnxnyK
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.q4KFhCaSxY
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state
+ rm /tmp/tmp.VcRNbnxnyK /tmp/tmp.q4KFhCaSxY
+ return 0
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/minio-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.F9aE9Z2sOe
++ mktemp
+ local LAST_ERR=/tmp/tmp.7VmynFX9Jx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/client.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/conf/minio-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F9aE9Z2sOe
secret/some-users created
deployment.apps/psmdb-client created
secret/minio-secret created
+ cat /tmp/tmp.7VmynFX9Jx
+ rm /tmp/tmp.F9aE9Z2sOe /tmp/tmp.7VmynFX9Jx
+ return 0
+ cluster=some-name
+ desc 'create first PSMDB cluster some-name'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1556-43640f06"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.UdXgPPybep
++ mktemp
+ local LAST_ERR=/tmp/tmp.XzMfoM6UR7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UdXgPPybep
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.XzMfoM6UR7
+ rm /tmp/tmp.UdXgPPybep /tmp/tmp.XzMfoM6UR7
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IwvjH8zvqr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N7CC7EaFSz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IwvjH8zvqr
++ cat /tmp/tmp.N7CC7EaFSz
++ rm /tmp/tmp.IwvjH8zvqr /tmp/tmp.N7CC7EaFSz
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready..........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C2tmcrjUTN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aCPi3KIoE3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C2tmcrjUTN
++ cat /tmp/tmp.aCPi3KIoE3
++ rm /tmp/tmp.C2tmcrjUTN /tmp/tmp.aCPi3KIoE3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.............................................
+ wait_for_running some-name-cfg 3 false
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=false
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NaGavNlbtk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZweGLobq3L
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NaGavNlbtk
++ cat /tmp/tmp.ZweGLobq3L
++ rm /tmp/tmp.NaGavNlbtk /tmp/tmp.ZweGLobq3L
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FXkU6kJLFA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wcYgxffXfv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FXkU6kJLFA
++ cat /tmp/tmp.wcYgxffXfv
++ rm /tmp/tmp.FXkU6kJLFA /tmp/tmp.wcYgxffXfv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ false == \t\r\u\e ]]
+ sleep 10
+ write_initial_data
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.pitr-physical-18132
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.42vEqfmlCi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YqAmQ93KvC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.42vEqfmlCi
++ cat /tmp/tmp.YqAmQ93KvC
++ rm /tmp/tmp.42vEqfmlCi /tmp/tmp.YqAmQ93KvC
++ return 0
+ local client_container=psmdb-client-7469665986-czs67
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FxxDOZonV5
++ mktemp
+ local LAST_ERR=/tmp/tmp.JpdwImCLBt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FxxDOZonV5
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-18132.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("efcc0d66-754a-4086-acce-c050b366b838") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.JpdwImCLBt
+ rm /tmp/tmp.FxxDOZonV5 /tmp/tmp.JpdwImCLBt
+ return 0
+ sleep 2
+ write_document
+ local cmp_postfix=
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-18132
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kS4cBUvvsE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AchE0ejs44
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kS4cBUvvsE
++ cat /tmp/tmp.AchE0ejs44
++ rm /tmp/tmp.kS4cBUvvsE /tmp/tmp.AchE0ejs44
++ return 0
+ local client_container=psmdb-client-7469665986-czs67
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cXQlwTAEmz
++ mktemp
+ local LAST_ERR=/tmp/tmp.8u362W281R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cXQlwTAEmz
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-18132.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("f0a14967-793e-4eff-89a3-eb001fe0870a") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.8u362W281R
+ rm /tmp/tmp.cXQlwTAEmz /tmp/tmp.8u362W281R
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-18132
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-18132 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.nIgdUfHeZP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1qS96pJ2G5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nIgdUfHeZP
++ cat /tmp/tmp.1qS96pJ2G5
++ rm /tmp/tmp.nIgdUfHeZP /tmp/tmp.1qS96pJ2G5
++ return 0
+ local client_container=psmdb-client-7469665986-czs67
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.cj7ZV808Ru
++ mktemp
+ local LAST_ERR=/tmp/tmp.xdSdQ0sUnN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-czs67 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cj7ZV808Ru
+ cat /tmp/tmp.xdSdQ0sUnN
+ rm /tmp/tmp.cj7ZV808Ru /tmp/tmp.xdSdQ0sUnN
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/compare/find.json /tmp/tmp.sLt1hO2aZC/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
some-name-rs0-0
+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
some-name-rs0-1
+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
some-name-rs0-2
+ wait_backup_agent some-name-rs1-0
+ local agent_pod=some-name-rs1-0
+ set +o xtrace
some-name-rs1-0
+ wait_backup_agent some-name-rs1-1
+ local agent_pod=some-name-rs1-1
+ set +o xtrace
some-name-rs1-1
+ wait_backup_agent some-name-rs1-2
+ local agent_pod=some-name-rs1-2
+ set +o xtrace
some-name-rs1-2...Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
Error from server (BadRequest): container "backup-agent" in pod "some-name-rs1-2" is waiting to start: PodInitializing
.
+ wait_backup_agent some-name-rs2-0
+ local agent_pod=some-name-rs2-0
+ set +o xtrace
some-name-rs2-0
+ wait_backup_agent some-name-rs2-1
+ local agent_pod=some-name-rs2-1
+ set +o xtrace
some-name-rs2-1
+ wait_backup_agent some-name-rs2-2
+ local agent_pod=some-name-rs2-2
+ set +o xtrace
some-name-rs2-2
+ backup_name_minio=backup-minio
+ sleep 360
+ run_backup backup-minio 1 physical
+ local name=backup-minio
+ local idx=1
+ local type=physical
+ desc 'run backup backup-minio-1'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-1
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/backup-minio.yml
+ /usr/bin/sed -e 's/type:/type: physical/'
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's/name:/name: backup-minio-1/'
+ local LAST_OUT=/tmp/tmp.gV8zQqlRUu
++ mktemp
+ local LAST_ERR=/tmp/tmp.trwNrY9HHX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gV8zQqlRUu
perconaservermongodbbackup.psmdb.percona.com/backup-minio-1 created
+ cat /tmp/tmp.trwNrY9HHX
+ rm /tmp/tmp.gV8zQqlRUu /tmp/tmp.trwNrY9HHX
+ return 0
+ wait_backup backup-minio-1
+ local backup_name=backup-minio-1
+ set +o xtrace
backup-minio-1..................
+ sleep 5
+ write_document -2nd
+ local cmp_postfix=-2nd
+ desc 'write initial data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write initial data, read from all
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.pitr-physical-18132
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HtAXzYSa8m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7SjSq2FM9C
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HtAXzYSa8m
++ cat /tmp/tmp.7SjSq2FM9C
++ rm /tmp/tmp.HtAXzYSa8m /tmp/tmp.7SjSq2FM9C
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.XHNF7pDEE7
++ mktemp
+ local LAST_ERR=/tmp/tmp.FQEkreyj1A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XHNF7pDEE7
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-18132.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e5cba698-efe4-4703-b61d-69f5e82a7f07") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.FQEkreyj1A
+ rm /tmp/tmp.XHNF7pDEE7 /tmp/tmp.FQEkreyj1A
+ return 0
+ minikube_sleep
+ sleep_time=10
+ [[ '' == 1 ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-18132 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-18132 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z75Y7VgXqa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hf5rYRUAq6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z75Y7VgXqa
++ cat /tmp/tmp.hf5rYRUAq6
++ rm /tmp/tmp.Z75Y7VgXqa /tmp/tmp.hf5rYRUAq6
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ycm4AoPMSB
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZMiMlOpUTm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Ycm4AoPMSB
+ cat /tmp/tmp.ZMiMlOpUTm
+ rm /tmp/tmp.Ycm4AoPMSB /tmp/tmp.ZMiMlOpUTm
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/compare/find-2nd.json /tmp/tmp.sLt1hO2aZC/find-2nd
++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ jq '.backups.snapshot[0].restoreTo'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dpz3yz4xlF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RW6FkfpcoL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dpz3yz4xlF
++ cat /tmp/tmp.RW6FkfpcoL
++ rm /tmp/tmp.dpz3yz4xlF /tmp/tmp.RW6FkfpcoL
++ return 0
+ backup_last_write=1716569408
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Hku2jY49Pt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hGtdTsJawF
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Hku2jY49Pt
+++ cat /tmp/tmp.hGtdTsJawF
+++ rm /tmp/tmp.Hku2jY49Pt /tmp/tmp.hGtdTsJawF
+++ return 0
++ echo null
+ last_chunk=null
+ retries=0
+ [[ null -gt 1716569408 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qF8278Kb60
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6GV1E99VGD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.qF8278Kb60
+++ cat /tmp/tmp.6GV1E99VGD
+++ rm /tmp/tmp.qF8278Kb60 /tmp/tmp.6GV1E99VGD
+++ return 0
++ echo null
+ last_chunk=null
+ retries=1
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.4QSpfqNgBu
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.crpMICp7GO
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.4QSpfqNgBu
+++ cat /tmp/tmp.crpMICp7GO
+++ rm /tmp/tmp.4QSpfqNgBu /tmp/tmp.crpMICp7GO
+++ return 0
++ echo null
+ last_chunk=null
+ retries=2
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PhmJ7euw5D
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OoWDC6LA1A
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.PhmJ7euw5D
+++ cat /tmp/tmp.OoWDC6LA1A
+++ rm /tmp/tmp.PhmJ7euw5D /tmp/tmp.OoWDC6LA1A
+++ return 0
++ echo null
+ last_chunk=null
+ retries=3
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.A1vZNshUDZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.30bY3EAUja
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.A1vZNshUDZ
+++ cat /tmp/tmp.30bY3EAUja
+++ rm /tmp/tmp.A1vZNshUDZ /tmp/tmp.30bY3EAUja
+++ return 0
++ echo null
+ last_chunk=null
+ retries=4
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bnWbeaEFxG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SwhC0NEiWX
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.bnWbeaEFxG
+++ cat /tmp/tmp.SwhC0NEiWX
+++ rm /tmp/tmp.bnWbeaEFxG /tmp/tmp.SwhC0NEiWX
+++ return 0
++ echo null
+ last_chunk=null
+ retries=5
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jBx4yST7hO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Q5h6cxfAQb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.jBx4yST7hO
+++ cat /tmp/tmp.Q5h6cxfAQb
+++ rm /tmp/tmp.jBx4yST7hO /tmp/tmp.Q5h6cxfAQb
+++ return 0
++ echo null
+ last_chunk=null
+ retries=6
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RcQsiH3Ee0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tYM5lTHDgN
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.RcQsiH3Ee0
+++ cat /tmp/tmp.tYM5lTHDgN
+++ rm /tmp/tmp.RcQsiH3Ee0 /tmp/tmp.tYM5lTHDgN
+++ return 0
++ echo null
+ last_chunk=null
+ retries=7
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tvjOZrmtD2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nBNqXX65oD
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tvjOZrmtD2
+++ cat /tmp/tmp.nBNqXX65oD
+++ rm /tmp/tmp.tvjOZrmtD2 /tmp/tmp.nBNqXX65oD
+++ return 0
++ echo null
+ last_chunk=null
+ retries=8
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iZahIn5ioU
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.A7RSVIyulL
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iZahIn5ioU
+++ cat /tmp/tmp.A7RSVIyulL
+++ rm /tmp/tmp.iZahIn5ioU /tmp/tmp.A7RSVIyulL
+++ return 0
++ echo null
+ last_chunk=null
+ retries=9
++ format_date null
++ local timestamp=null
+++ TZ=UTC
+++ /usr/bin/date -d@null '+%Y-%m-%d %H:%M:%S'
/usr/bin/date: invalid date ‘@null’
++ echo
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk () to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ null -gt 1716569408 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hHxcUyMWI1
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.hfkjCVFoyU
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.hHxcUyMWI1
+++ cat /tmp/tmp.hfkjCVFoyU
+++ rm /tmp/tmp.hHxcUyMWI1 /tmp/tmp.hfkjCVFoyU
+++ return 0
++ echo 1716569546
+ last_chunk=1716569546
+ retries=10
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569408
++ local timestamp=1716569408
+++ TZ=UTC
+++ /usr/bin/date -d@1716569408 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:50:08
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than last write (2024-05-24 16:50:08)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than last write (2024-05-24 16:50:08)
+ sleep 10
+ [[ 1716569546 -gt 1716569408 ]]
+ check_recovery backup-minio-1 date 1716569546 -2nd some-name
+ local backup_name=backup-minio-1
+ local restore_type=date
+ local restore_date=1716569546
+ local cmp_postfix=-2nd
+ local cluster_name=some-name
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.8aSRacTN75
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K7LOguMeUp
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.8aSRacTN75
+++ cat /tmp/tmp.K7LOguMeUp
+++ rm /tmp/tmp.8aSRacTN75 /tmp/tmp.K7LOguMeUp
+++ return 0
++ echo 1716569546
+ local latest_ts=1716569546
+ desc 'write more data before restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by date
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-18132
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EzjQr74UVG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MwOgU9jTeK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EzjQr74UVG
++ cat /tmp/tmp.MwOgU9jTeK
++ rm /tmp/tmp.EzjQr74UVG /tmp/tmp.MwOgU9jTeK
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.t1CWebNx2P
++ mktemp
+ local LAST_ERR=/tmp/tmp.I4w2TLMV7t
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.t1CWebNx2P
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-18132.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("de661a52-6af9-4780-94e5-fb0112fc6569") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.I4w2TLMV7t
+ rm /tmp/tmp.t1CWebNx2P /tmp/tmp.I4w2TLMV7t
+ return 0
+ [[ -n 1716569546 ]]
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ desc 'Restoring to time 2024-05-24 16:52:26'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to time 2024-05-24 16:52:26
-----------------------------------------------------------------------------------

+ retries=0
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Q7WkecMhAm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.tBNAQLFOL8
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.Q7WkecMhAm
+++ cat /tmp/tmp.tBNAQLFOL8
+++ rm /tmp/tmp.Q7WkecMhAm /tmp/tmp.tBNAQLFOL8
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=1
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iX4iC2SwpL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.GMTSo96ug4
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.iX4iC2SwpL
+++ cat /tmp/tmp.GMTSo96ug4
+++ rm /tmp/tmp.iX4iC2SwpL /tmp/tmp.GMTSo96ug4
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=2
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XgYGwcEafw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ragN5qVt7E
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.XgYGwcEafw
+++ cat /tmp/tmp.ragN5qVt7E
+++ rm /tmp/tmp.XgYGwcEafw /tmp/tmp.ragN5qVt7E
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=3
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dqXCAeGWo4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3eH908s85U
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.dqXCAeGWo4
+++ cat /tmp/tmp.3eH908s85U
+++ rm /tmp/tmp.dqXCAeGWo4 /tmp/tmp.3eH908s85U
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=4
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QzJsqbymix
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2RPrI6ahXk
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.QzJsqbymix
+++ cat /tmp/tmp.2RPrI6ahXk
+++ rm /tmp/tmp.QzJsqbymix /tmp/tmp.2RPrI6ahXk
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=5
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2sy6F4BlsM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7NWzyP3f7z
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.2sy6F4BlsM
+++ cat /tmp/tmp.7NWzyP3f7z
+++ rm /tmp/tmp.2sy6F4BlsM /tmp/tmp.7NWzyP3f7z
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=6
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.17QDsBSjDU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.K4T8MN59js
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.17QDsBSjDU
+++ cat /tmp/tmp.K4T8MN59js
+++ rm /tmp/tmp.17QDsBSjDU /tmp/tmp.K4T8MN59js
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=7
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ByniYkyGWm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.abhVZ4Ci1D
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.ByniYkyGWm
+++ cat /tmp/tmp.abhVZ4Ci1D
+++ rm /tmp/tmp.ByniYkyGWm /tmp/tmp.abhVZ4Ci1D
+++ return 0
++ echo 1716569546
+ latest_ts=1716569546
+ retries=8
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:26) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569546 -gt 1716569546 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.pX88Zn56IC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vHdoy3R1PP
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.pX88Zn56IC
+++ cat /tmp/tmp.vHdoy3R1PP
+++ rm /tmp/tmp.pX88Zn56IC /tmp/tmp.vHdoy3R1PP
+++ return 0
++ echo 1716569550
+ latest_ts=1716569550
+ retries=9
++ format_date 1716569550
++ local timestamp=1716569550
+++ TZ=UTC
+++ /usr/bin/date -d@1716569550 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:30
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:52:26
+ echo 'Waiting for last oplog chunk (2024-05-24 16:52:30) to be greater than restore target (2024-05-24 16:52:26)'
Waiting for last oplog chunk (2024-05-24 16:52:30) to be greater than restore target (2024-05-24 16:52:26)
+ sleep 10
+ [[ 1716569550 -gt 1716569546 ]]
+ desc 'check restore by date'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by date
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-1/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-1/'
+ kubectl_bin apply -f -
+ '[' -z 1716569546 ']'
+ /usr/bin/sed -e 's/type:/type: date/'
++ mktemp
++ format_date 1716569546
++ local timestamp=1716569546
+++ TZ=UTC
+++ /usr/bin/date -d@1716569546 '+%Y-%m-%d %H:%M:%S'
+ local LAST_OUT=/tmp/tmp.eGUHy1M383
++ mktemp
+ local LAST_ERR=/tmp/tmp.bcvR8NkCy0
+ local exit_status=0
++ echo 2024-05-24 16:52:26
+ local timeout=4
+ /usr/bin/sed -e 's/date:/date: 2024-05-24 16:52:26/'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eGUHy1M383
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-1 created
+ cat /tmp/tmp.bcvR8NkCy0
+ rm /tmp/tmp.eGUHy1M383 /tmp/tmp.bcvR8NkCy0
+ return 0
+ wait_restore backup-minio-1 some-name requested 0 900
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ set +o xtrace
waiting psmdb-restore/backup-minio-1 to reach requested state.........................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ wait_restore backup-minio-1 some-name ready 0 1600
+ local backup_name=backup-minio-1
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ set +o xtrace
waiting psmdb-restore/backup-minio-1 to reach ready state...........................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ set -o xtrace
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready........................OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NqPtvrxEIT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M7mUGMUwfS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NqPtvrxEIT
++ cat /tmp/tmp.M7mUGMUwfS
++ rm /tmp/tmp.NqPtvrxEIT /tmp/tmp.M7mUGMUwfS
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.s2gB8Hveme
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f5ovmTI0BD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.s2gB8Hveme
++ cat /tmp/tmp.f5ovmTI0BD
++ rm /tmp/tmp.s2gB8Hveme /tmp/tmp.f5ovmTI0BD
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-18132 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-18132 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gKfdgxx6wX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fdQ7cDqHN5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gKfdgxx6wX
++ cat /tmp/tmp.fdQ7cDqHN5
++ rm /tmp/tmp.gKfdgxx6wX /tmp/tmp.fdQ7cDqHN5
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vGsT7r84Kc
++ mktemp
+ local LAST_ERR=/tmp/tmp.dWWjAoTazG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vGsT7r84Kc
+ cat /tmp/tmp.dWWjAoTazG
+ rm /tmp/tmp.vGsT7r84Kc /tmp/tmp.dWWjAoTazG
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/compare/find-2nd.json /tmp/tmp.sLt1hO2aZC/find-2nd
+ run_backup backup-minio 3 physical
+ local name=backup-minio
+ local idx=3
+ local type=physical
+ desc 'run backup backup-minio-3'
+ set +o xtrace


-----------------------------------------------------------------------------------
run backup backup-minio-3
-----------------------------------------------------------------------------------

+ /usr/bin/sed -e 's/name:/name: backup-minio-3/'
+ /usr/bin/sed -e 's/type:/type: physical/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/backup-minio.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gu3ziIvmnZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.U5mg6mONy1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gu3ziIvmnZ
perconaservermongodbbackup.psmdb.percona.com/backup-minio-3 created
+ cat /tmp/tmp.U5mg6mONy1
+ rm /tmp/tmp.gu3ziIvmnZ /tmp/tmp.U5mg6mONy1
+ return 0
+ wait_backup backup-minio-3
+ local backup_name=backup-minio-3
+ set +o xtrace
backup-minio-3....................
+ sleep 5
+ compare_latest_restorable_time some-name-rs0 backup-minio-3
+ local cluster=some-name-rs0
+ local backup_name=backup-minio-3
+ local latest_restorable_time
+ local backup_time
++ get_latest_restorable_time some-name-rs0
++ local cluster=some-name-rs0
++ local first_timestamp
++ local second_timestamp
++ local retry=0
++ [[ '' != '' ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LwtwpIP3CE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XYbnvAzdAf
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.LwtwpIP3CE
+++ cat /tmp/tmp.XYbnvAzdAf
+++ rm /tmp/tmp.LwtwpIP3CE /tmp/tmp.XYbnvAzdAf
+++ return 0
++ first_timestamp=1716569670
++ sleep 5
++ [[ 1716569670 != '' ]]
++ [[ 1716569670 != \n\u\l\l ]]
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OSyMUSULMt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EqVkNGArd3
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.OSyMUSULMt
+++ cat /tmp/tmp.EqVkNGArd3
+++ rm /tmp/tmp.OSyMUSULMt /tmp/tmp.EqVkNGArd3
+++ return 0
++ second_timestamp=1716569670
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 1716569670 != '' ]]
++ [[ 1716569670 != \n\u\l\l ]]
++ [[ 1716569670 == 1716569670 ]]
++ /usr/bin/date -u -d @1716569670 +%Y-%m-%dT%H:%M:%SZ
+ latest_restorable_time=2024-05-24T16:54:30Z
++ get_latest_restorable_time_from_backup_object backup-minio-3
++ local backup_name=backup-minio-3
++ local latestRestorableTime
++ local retry=0
++ [[ '' != '' ]]
++ sleep 5
+++ kubectl_bin get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fh7l4NKnyt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kxJT3qmmI6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get psmdb-backup backup-minio-3 -o 'jsonpath={.status.latestRestorableTime}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.fh7l4NKnyt
+++ cat /tmp/tmp.kxJT3qmmI6
+++ rm /tmp/tmp.fh7l4NKnyt /tmp/tmp.kxJT3qmmI6
+++ return 0
++ latestRestorableTime=2024-05-24T16:54:30Z
++ let retry+=1
++ [[ 1 -gt 30 ]]
++ [[ 2024-05-24T16:54:30Z != '' ]]
++ [[ 2024-05-24T16:54:30Z != \n\u\l\l ]]
++ echo 2024-05-24T16:54:30Z
+ backup_time=2024-05-24T16:54:30Z
+ [[ 2024-05-24T16:54:30Z != \2\0\2\4\-\0\5\-\2\4\T\1\6\:\5\4\:\3\0\Z ]]
+ check_recovery backup-minio-3 latest '' -3rd some-name
+ local backup_name=backup-minio-3
+ local restore_type=latest
+ local restore_date=
+ local cmp_postfix=-3rd
+ local cluster_name=some-name
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5e6o8Feagr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9PKCT6Daxb
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.5e6o8Feagr
+++ cat /tmp/tmp.9PKCT6Daxb
+++ rm /tmp/tmp.5e6o8Feagr /tmp/tmp.9PKCT6Daxb
+++ return 0
++ echo 1716569670
+ local latest_ts=1716569670
+ desc 'write more data before restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
write more data before restore by latest
-----------------------------------------------------------------------------------

+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.pitr-physical-18132
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VBIwukQZ57
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fvZMHWUOjz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VBIwukQZ57
++ cat /tmp/tmp.fvZMHWUOjz
++ rm /tmp/tmp.VBIwukQZ57 /tmp/tmp.fvZMHWUOjz
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YsAINCZ1oD
++ mktemp
+ local LAST_ERR=/tmp/tmp.mQhktoyhmN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YsAINCZ1oD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.pitr-physical-18132.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a481456d-7703-4ab8-8858-853696867606") }
Percona Server for MongoDB server version: v7.0.8-5
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.mQhktoyhmN
+ rm /tmp/tmp.YsAINCZ1oD /tmp/tmp.mQhktoyhmN
+ return 0
+ [[ -n '' ]]
+ desc 'Restoring to latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restoring to latest
-----------------------------------------------------------------------------------

++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CGa8KRbTEB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.g95LXeM41A
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.CGa8KRbTEB
+++ cat /tmp/tmp.g95LXeM41A
+++ rm /tmp/tmp.CGa8KRbTEB /tmp/tmp.g95LXeM41A
+++ return 0
++ echo 1716569670
+ local current_ts=1716569670
+ retries=0
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 0 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kTb5izJ1F8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.X0jFKmSHhY
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.kTb5izJ1F8
+++ cat /tmp/tmp.X0jFKmSHhY
+++ rm /tmp/tmp.kTb5izJ1F8 /tmp/tmp.X0jFKmSHhY
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=1
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 1 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GK2ifGG9sB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OadfAR6fUi
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.GK2ifGG9sB
+++ cat /tmp/tmp.OadfAR6fUi
+++ rm /tmp/tmp.GK2ifGG9sB /tmp/tmp.OadfAR6fUi
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=2
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 2 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lANom1qagm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.AOtAY9abqw
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.lANom1qagm
+++ cat /tmp/tmp.AOtAY9abqw
+++ rm /tmp/tmp.lANom1qagm /tmp/tmp.AOtAY9abqw
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=3
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 3 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SyRTBPvMbf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cdhywZ1yEc
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.SyRTBPvMbf
+++ cat /tmp/tmp.cdhywZ1yEc
+++ rm /tmp/tmp.SyRTBPvMbf /tmp/tmp.cdhywZ1yEc
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=4
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 4 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7TCgqkJfKf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zVIC4dZn03
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.7TCgqkJfKf
+++ cat /tmp/tmp.zVIC4dZn03
+++ rm /tmp/tmp.7TCgqkJfKf /tmp/tmp.zVIC4dZn03
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=5
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 5 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FbKgGirtBI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KPJYUqMHVn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FbKgGirtBI
+++ cat /tmp/tmp.KPJYUqMHVn
+++ rm /tmp/tmp.FbKgGirtBI /tmp/tmp.KPJYUqMHVn
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=6
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 6 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tLaxA8h6zZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.912mXX54n6
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.tLaxA8h6zZ
+++ cat /tmp/tmp.912mXX54n6
+++ rm /tmp/tmp.tLaxA8h6zZ /tmp/tmp.912mXX54n6
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=7
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 7 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FpQA6dd0PO
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oe1204ooAn
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.FpQA6dd0PO
+++ cat /tmp/tmp.oe1204ooAn
+++ rm /tmp/tmp.FpQA6dd0PO /tmp/tmp.oe1204ooAn
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=8
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 8 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sehtaTuMzr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.u9JgPqeqAV
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.sehtaTuMzr
+++ cat /tmp/tmp.u9JgPqeqAV
+++ rm /tmp/tmp.sehtaTuMzr /tmp/tmp.u9JgPqeqAV
+++ return 0
++ echo 1716569670
+ latest_ts=1716569670
+ retries=9
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 16:54:30) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716569670 -gt 1716569670 ]]
+ [[ 9 -gt 30 ]]
++ get_latest_oplog_chunk_ts some-name
++ local cluster=some-name
+++ kubectl_bin exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ jq '.backups.pitrChunks.pitrChunks | last | .range.end'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.EZlp6Jm7FZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.osZ1HOjFYJ
+++ local exit_status=0
+++ local timeout=4
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl exec some-name-rs0-0 -c backup-agent -- pbm status -o json
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 -a -n 1 ']'
+++ break
+++ cat /tmp/tmp.EZlp6Jm7FZ
+++ cat /tmp/tmp.osZ1HOjFYJ
+++ rm /tmp/tmp.EZlp6Jm7FZ /tmp/tmp.osZ1HOjFYJ
+++ return 0
++ echo 1716570525
+ latest_ts=1716570525
+ retries=10
++ format_date 1716570525
++ local timestamp=1716570525
+++ TZ=UTC
+++ /usr/bin/date -d@1716570525 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 17:08:45
++ format_date 1716569670
++ local timestamp=1716569670
+++ TZ=UTC
+++ /usr/bin/date -d@1716569670 '+%Y-%m-%d %H:%M:%S'
++ echo 2024-05-24 16:54:30
+ echo 'Waiting for last oplog chunk (2024-05-24 17:08:45) to be greater than starting chunk (2024-05-24 16:54:30)'
Waiting for last oplog chunk (2024-05-24 17:08:45) to be greater than starting chunk (2024-05-24 16:54:30)
+ sleep 10
+ [[ 1716570525 -gt 1716569670 ]]
+ desc 'check restore by latest'
+ set +o xtrace


-----------------------------------------------------------------------------------
check restore by latest
-----------------------------------------------------------------------------------

+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-3/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-3/'
+ kubectl_bin apply -f -
+ '[' -z '' ']'
+ /usr/bin/sed -e /date:/d
+ /usr/bin/sed -e 's/type:/type: latest/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.P0ZhZMDkIC
++ mktemp
+ local LAST_ERR=/tmp/tmp.9OAHSPs2I9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P0ZhZMDkIC
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-3 created
+ cat /tmp/tmp.9OAHSPs2I9
+ rm /tmp/tmp.P0ZhZMDkIC /tmp/tmp.9OAHSPs2I9
+ return 0
+ wait_restore backup-minio-3 some-name requested 0 900
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=900
+ set +o xtrace
waiting psmdb-restore/backup-minio-3 to reach requested state............................................................................................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ wait_restore backup-minio-3 some-name ready 0 1600
+ local backup_name=backup-minio-3
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=1600
+ set +o xtrace
waiting psmdb-restore/backup-minio-3 to reach ready state...................................................................................................
+ '[' 0 -eq 1 ']'
+ echo

+ set -o xtrace
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready..............................OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.11SPKBsKHm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K7CvJY8DZl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.11SPKBsKHm
++ cat /tmp/tmp.K7CvJY8DZl
++ rm /tmp/tmp.11SPKBsKHm /tmp/tmp.K7CvJY8DZl
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.........OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XUubSNtR1B
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YcrT58iAWG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XUubSNtR1B
++ cat /tmp/tmp.YcrT58iAWG
++ rm /tmp/tmp.XUubSNtR1B /tmp/tmp.YcrT58iAWG
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ sleep 10
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.pitr-physical-18132 -3rd
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local postfix=-3rd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.pitr-physical-18132 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.pitr-physical-18132
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QUHXoP2VWP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Zb3yAueh4y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QUHXoP2VWP
++ cat /tmp/tmp.Zb3yAueh4y
++ rm /tmp/tmp.QUHXoP2VWP /tmp/tmp.Zb3yAueh4y
++ return 0
+ local client_container=psmdb-client-7469665986-84r9l
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tTu8rjoQq7
++ mktemp
+ local LAST_ERR=/tmp/tmp.1hKAbUfcrB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-7469665986-84r9l -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.pitr-physical-18132.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tTu8rjoQq7
+ cat /tmp/tmp.1hKAbUfcrB
+ rm /tmp/tmp.tTu8rjoQq7 /tmp/tmp.1hKAbUfcrB
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/e2e-tests/pitr-physical/compare/find-3rd.json /tmp/tmp.sLt1hO2aZC/find-3rd
+ destroy pitr-physical-18132
+ local namespace=pitr-physical-18132
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.pPCEZQwJ6f
++ mktemp
+ local LAST_ERR=/tmp/tmp.UQXqA889We
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pPCEZQwJ6f
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.UQXqA889We
+ rm /tmp/tmp.pPCEZQwJ6f /tmp/tmp.UQXqA889We
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.nu1m1gzD6V
++ mktemp
+ local LAST_ERR=/tmp/tmp.iwGIwLqd7p
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nu1m1gzD6V
+ cat /tmp/tmp.iwGIwLqd7p
+ rm /tmp/tmp.nu1m1gzD6V /tmp/tmp.iwGIwLqd7p
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.xmv91oOEQm
++ mktemp
+ local LAST_ERR=/tmp/tmp.eFFwO0qlqN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xmv91oOEQm
+ cat /tmp/tmp.eFFwO0qlqN
+ rm /tmp/tmp.xmv91oOEQm /tmp/tmp.eFFwO0qlqN
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.fuyUo3Kf4N
++ mktemp
+ local LAST_ERR=/tmp/tmp.gAsccTkK6w
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fuyUo3Kf4N
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com condition met
+ cat /tmp/tmp.gAsccTkK6w
+ rm /tmp/tmp.fuyUo3Kf4N /tmp/tmp.gAsccTkK6w
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.hQXbvgTYPE
++ mktemp
+ local LAST_ERR=/tmp/tmp.7Xwwar3cCm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1556/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hQXbvgTYPE
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.7Xwwar3cCm
+ rm /tmp/tmp.hQXbvgTYPE /tmp/tmp.7Xwwar3cCm
+ return 0
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.5/cert-manager.yaml
+ :
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace pitr-physical-18132
+ rm -rf /tmp/tmp.sLt1hO2aZC
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.PmVNUuCUwl
+ local LAST_OUT=/tmp/tmp.QbhU1OLDca
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.cT7mu9r1XX
+ local exit_status=0
+ local timeout=4
+ local LAST_ERR=/tmp/tmp.3bwivQzDGy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pitr-physical-18132
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator