Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/logs/scheduled-backup.log
WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.31) and server (1.27) exceeds the supported minor version skew of +/-1
+ create_infra scheduled-backup-11064
+ local ns=scheduled-backup-11064
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.JkDAEJvM58
++ mktemp
+ local LAST_ERR=/tmp/tmp.XRT9g7z5Q5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JkDAEJvM58
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.XRT9g7z5Q5
+ rm /tmp/tmp.JkDAEJvM58 /tmp/tmp.XRT9g7z5Q5
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-25889 cron-some-name-20240912232400-f52hn --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912232400-f52hn patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-25889 cron-some-name-20240912232400-q6rf7 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912232400-q6rf7 patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-25889 cron-some-name-20240912232400-z6fzp --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912232400-z6fzp patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.8Be5QswY5w
++ mktemp
+ local LAST_ERR=/tmp/tmp.ahAYCfuN8Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8Be5QswY5w
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.ahAYCfuN8Q
+ rm /tmp/tmp.8Be5QswY5w /tmp/tmp.ahAYCfuN8Q
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.ORXnKqm0gr
++ mktemp
+ local LAST_ERR=/tmp/tmp.xuELWPOwaX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ORXnKqm0gr
+ cat /tmp/tmp.xuELWPOwaX
+ rm /tmp/tmp.ORXnKqm0gr /tmp/tmp.xuELWPOwaX
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.vmuPRlPkaO
++ mktemp
+ local LAST_ERR=/tmp/tmp.EWysBRCSds
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vmuPRlPkaO
+ cat /tmp/tmp.EWysBRCSds
+ rm /tmp/tmp.vmuPRlPkaO /tmp/tmp.EWysBRCSds
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.kcILkqZi1i
++ mktemp
+ local LAST_ERR=/tmp/tmp.eVe5ImR2ET
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kcILkqZi1i
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.eVe5ImR2ET
+ rm /tmp/tmp.kcILkqZi1i /tmp/tmp.eVe5ImR2ET
+ return 0
+ check_crd_for_deletion PR-1641-df7592c8
+ local git_tag=PR-1641-df7592c8
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1641-df7592c8/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vYrispRAt0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cHyR5sgXbS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.vYrispRAt0
++ cat /tmp/tmp.cHyR5sgXbS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.vYrispRAt0
++ cat /tmp/tmp.cHyR5sgXbS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.vYrispRAt0
++ cat /tmp/tmp.cHyR5sgXbS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.vYrispRAt0
++ cat /tmp/tmp.cHyR5sgXbS
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.vYrispRAt0 /tmp/tmp.cHyR5sgXbS
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.SejUUNkB7r
+ kubectl_bin get ns
++ mktemp
+ xargs kubectl delete ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.C2cCeUv1kE
+ local LAST_ERR=/tmp/tmp.k6bWh9aJTs
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ local LAST_ERR=/tmp/tmp.7YC388kD3E
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C2cCeUv1kE
+ cat /tmp/tmp.7YC388kD3E
+ rm /tmp/tmp.C2cCeUv1kE /tmp/tmp.7YC388kD3E
+ return 0
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "scheduled-backup-25889" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SejUUNkB7r
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.k6bWh9aJTs
+ rm /tmp/tmp.SejUUNkB7r /tmp/tmp.k6bWh9aJTs
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.xqNmX3464Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.d66BLTLwu5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xqNmX3464Q
+ cat /tmp/tmp.d66BLTLwu5
+ rm /tmp/tmp.xqNmX3464Q /tmp/tmp.d66BLTLwu5
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6cxzWdlsmK
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ke3j0x9UZG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6cxzWdlsmK
namespace/psmdb-operator created
+ cat /tmp/tmp.Ke3j0x9UZG
+ rm /tmp/tmp.6cxzWdlsmK /tmp/tmp.Ke3j0x9UZG
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WQyxRIpSN3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0EhPDMZ9tN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WQyxRIpSN3
++ cat /tmp/tmp.0EhPDMZ9tN
++ rm /tmp/tmp.WQyxRIpSN3 /tmp/tmp.0EhPDMZ9tN
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wZoIJhy5br
++ mktemp
+ local LAST_ERR=/tmp/tmp.E9l1Hq9Eb1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wZoIJhy5br
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3" modified.
+ cat /tmp/tmp.E9l1Hq9Eb1
+ rm /tmp/tmp.wZoIJhy5br /tmp/tmp.E9l1Hq9Eb1
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2U1Zw1nhtx
++ mktemp
+ local LAST_ERR=/tmp/tmp.KpsvdlIxRH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2U1Zw1nhtx
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.KpsvdlIxRH
+ rm /tmp/tmp.2U1Zw1nhtx /tmp/tmp.KpsvdlIxRH
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ local LAST_OUT=/tmp/tmp.UXeU8jAkv1
++ mktemp
+ local LAST_ERR=/tmp/tmp.Mbdi0jwrEs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-rbac.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UXeU8jAkv1
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.Mbdi0jwrEs
+ rm /tmp/tmp.UXeU8jAkv1 /tmp/tmp.Mbdi0jwrEs
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.r1kuYBvJay
++ mktemp
+ local LAST_ERR=/tmp/tmp.fDxuEVgcBN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.r1kuYBvJay
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.fDxuEVgcBN
+ rm /tmp/tmp.r1kuYBvJay /tmp/tmp.fDxuEVgcBN
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x3FqhgT1fB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rg6ZTP8DXa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.x3FqhgT1fB
++ cat /tmp/tmp.Rg6ZTP8DXa
++ rm /tmp/tmp.x3FqhgT1fB /tmp/tmp.Rg6ZTP8DXa
++ return 0
+ wait_pod percona-server-mongodb-operator-645868bc47-4h26c
+ local pod=percona-server-mongodb-operator-645868bc47-4h26c
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-645868bc47-4h26c to be ready.OK
+ create_namespace scheduled-backup-11064
+ local namespace=scheduled-backup-11064
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces scheduled-backup-11064'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces scheduled-backup-11064
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace scheduled-backup-11064 --ignore-not-found
++ mktemp
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.kup2Bcrupt
+ kubectl_bin get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.bfAcVyYlz8
+ local exit_status=0
+ local timeout=4
++ mktemp
+ local LAST_OUT=/tmp/tmp.pXO2kSeDoz
++ mktemp
+ local LAST_ERR=/tmp/tmp.cn1GzvGJM3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scheduled-backup-11064 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pXO2kSeDoz
+ cat /tmp/tmp.cn1GzvGJM3
+ rm /tmp/tmp.pXO2kSeDoz /tmp/tmp.cn1GzvGJM3
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kup2Bcrupt
+ cat /tmp/tmp.bfAcVyYlz8
+ rm /tmp/tmp.kup2Bcrupt /tmp/tmp.bfAcVyYlz8
+ return 0
+ kubectl_bin wait --for=delete namespace scheduled-backup-11064
++ mktemp
+ local LAST_OUT=/tmp/tmp.KAiccYZlfz
++ mktemp
+ local LAST_ERR=/tmp/tmp.kcoZ7WgboH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace scheduled-backup-11064
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KAiccYZlfz
+ cat /tmp/tmp.kcoZ7WgboH
+ rm /tmp/tmp.KAiccYZlfz /tmp/tmp.kcoZ7WgboH
+ return 0
+ desc 'create namespace scheduled-backup-11064'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace scheduled-backup-11064
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace scheduled-backup-11064
++ mktemp
+ local LAST_OUT=/tmp/tmp.oHNuqXjmm8
++ mktemp
+ local LAST_ERR=/tmp/tmp.jSiaRlOu2H
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace scheduled-backup-11064
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oHNuqXjmm8
namespace/scheduled-backup-11064 created
+ cat /tmp/tmp.jSiaRlOu2H
+ rm /tmp/tmp.oHNuqXjmm8 /tmp/tmp.jSiaRlOu2H
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9FGiGp1y0s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uP5JuWLHCA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9FGiGp1y0s
++ cat /tmp/tmp.uP5JuWLHCA
++ rm /tmp/tmp.9FGiGp1y0s /tmp/tmp.uP5JuWLHCA
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3 --namespace=scheduled-backup-11064
++ mktemp
+ local LAST_OUT=/tmp/tmp.zYuF8CIqLR
++ mktemp
+ local LAST_ERR=/tmp/tmp.7GOsEVeWUD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3 --namespace=scheduled-backup-11064
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zYuF8CIqLR
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1641-df7592c8-4-cluster3" modified.
+ cat /tmp/tmp.7GOsEVeWUD
+ rm /tmp/tmp.zYuF8CIqLR /tmp/tmp.7GOsEVeWUD
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Sep 12 23:30:43 2024
NAMESPACE: scheduled-backup-11064
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.scheduled-backup-11064.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-11064 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-11064

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-11064 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-11064 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.H1DsbnJYoj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yKx7zjtMa0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.H1DsbnJYoj
++ cat /tmp/tmp.yKx7zjtMa0
++ rm /tmp/tmp.H1DsbnJYoj /tmp/tmp.yKx7zjtMa0
++ return 0
+ MINIO_POD=minio-service-6ff7647778-thwwq
+ wait_pod minio-service-6ff7647778-thwwq
+ local pod=minio-service-6ff7647778-thwwq
+ set +o xtrace
waiting for pod/minio-service-6ff7647778-thwwq to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-11064.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.bItpREFvcd
++ mktemp
+ local LAST_ERR=/tmp/tmp.d2JXOfBsuh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-11064.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bItpREFvcd
service/minio-service created
+ cat /tmp/tmp.d2JXOfBsuh
+ rm /tmp/tmp.bItpREFvcd /tmp/tmp.d2JXOfBsuh
+ return 0
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.mQsLSEPIrC
++ mktemp
+ local LAST_ERR=/tmp/tmp.SG6tHFlWDO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mQsLSEPIrC
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.SG6tHFlWDO
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-11064
+ rm /tmp/tmp.mQsLSEPIrC /tmp/tmp.SG6tHFlWDO
+ return 0
+ label_node
++ kubectl_bin get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
++ head -n1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PJRSsyYrIg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E14kuk7o50
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PJRSsyYrIg
++ cat /tmp/tmp.E14kuk7o50
++ rm /tmp/tmp.PJRSsyYrIg /tmp/tmp.E14kuk7o50
++ return 0
+ LABELED_NODE=gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09
+ desc 'add labels'
+ set +o xtrace


-----------------------------------------------------------------------------------
add labels
-----------------------------------------------------------------------------------

+ kubectl_bin label nodes gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 backupWorker=True --overwrite
++ mktemp
+ local LAST_OUT=/tmp/tmp.StIlxJAkqN
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZIFhDX9vo7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label nodes gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 backupWorker=True --overwrite
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.StIlxJAkqN
node/gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 labeled
+ cat /tmp/tmp.ZIFhDX9vo7
+ rm /tmp/tmp.StIlxJAkqN /tmp/tmp.ZIFhDX9vo7
+ return 0
+ desc 'create PriorityClass'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PriorityClass
-----------------------------------------------------------------------------------

+ cat -
+ kubectl apply -f -
priorityclass.scheduling.k8s.io/high-priority configured
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tG4z6jBrad
++ mktemp
+ local LAST_ERR=/tmp/tmp.iGQ35IodNC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tG4z6jBrad
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.iGQ35IodNC
+ rm /tmp/tmp.tG4z6jBrad /tmp/tmp.iGQ35IodNC
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.KfGSmFnFak
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Z9Sl8r9q9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KfGSmFnFak
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.2Z9Sl8r9q9
+ rm /tmp/tmp.KfGSmFnFak /tmp/tmp.2Z9Sl8r9q9
+ return 0
+ cluster=some-name-rs0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.5.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0qbE8Vw94B
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.yZQvJRURsD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0qbE8Vw94B
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.yZQvJRURsD
+ rm /tmp/tmp.0qbE8Vw94B /tmp/tmp.yZQvJRURsD
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.............OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tLgF5DCBve
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PZcAR5eQmo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tLgF5DCBve
++ cat /tmp/tmp.PZcAR5eQmo
++ rm /tmp/tmp.tLgF5DCBve /tmp/tmp.PZcAR5eQmo
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8W17cxTNar
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UaySCfepqv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8W17cxTNar
++ cat /tmp/tmp.UaySCfepqv
++ rm /tmp/tmp.8W17cxTNar /tmp/tmp.UaySCfepqv
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness....
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.ZIdgDFXkGo/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("scheduled-backup-11064", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.No2QXI1LjU
++ mktemp
+ local LAST_ERR=/tmp/tmp.ckiIPZXwO8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.No2QXI1LjU
+ cat /tmp/tmp.ckiIPZXwO8
+ rm /tmp/tmp.No2QXI1LjU /tmp/tmp.ckiIPZXwO8
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.ZIdgDFXkGo/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.27 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.ZIdgDFXkGo/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.ZIdgDFXkGo/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.ZIdgDFXkGo/statefulset_some-name-rs0.yml
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-11064
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mtrLmjLzJ2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DXc0QlZvaC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mtrLmjLzJ2
++ cat /tmp/tmp.DXc0QlZvaC
++ rm /tmp/tmp.mtrLmjLzJ2 /tmp/tmp.DXc0QlZvaC
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FpRkQEc08U
++ mktemp
+ local LAST_ERR=/tmp/tmp.pvgCaTreJT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FpRkQEc08U
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("8860f366-1024-40e4-8bdd-5e7bb9f4c0b0") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.pvgCaTreJT
+ rm /tmp/tmp.FpRkQEc08U /tmp/tmp.pvgCaTreJT
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A2h5MeWFZ9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5jyFMV4dS2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.A2h5MeWFZ9
++ cat /tmp/tmp.5jyFMV4dS2
++ rm /tmp/tmp.A2h5MeWFZ9 /tmp/tmp.5jyFMV4dS2
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qUZpoy1UQD
++ mktemp
+ local LAST_ERR=/tmp/tmp.2lmrOklGuK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qUZpoy1UQD
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("e5b72c2d-abf3-4b0a-b098-326136af79d6") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.2lmrOklGuK
+ rm /tmp/tmp.qUZpoy1UQD /tmp/tmp.2lmrOklGuK
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yo5XEcdmfN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V6gzrn5CnF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Yo5XEcdmfN
++ cat /tmp/tmp.V6gzrn5CnF
++ rm /tmp/tmp.Yo5XEcdmfN /tmp/tmp.V6gzrn5CnF
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LAVAhsyDlQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.tlg3LbJ3lu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LAVAhsyDlQ
+ cat /tmp/tmp.tlg3LbJ3lu
+ rm /tmp/tmp.LAVAhsyDlQ /tmp/tmp.tlg3LbJ3lu
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ibsj8yEnmM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pR6JHMIOkU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ibsj8yEnmM
++ cat /tmp/tmp.pR6JHMIOkU
++ rm /tmp/tmp.Ibsj8yEnmM /tmp/tmp.pR6JHMIOkU
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ADYIXHIQIT
++ mktemp
+ local LAST_ERR=/tmp/tmp.eOfStKKtAV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ADYIXHIQIT
+ cat /tmp/tmp.eOfStKKtAV
+ rm /tmp/tmp.ADYIXHIQIT /tmp/tmp.eOfStKKtAV
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GzsvsEs3W2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.coAT1nJDMR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.GzsvsEs3W2
++ cat /tmp/tmp.coAT1nJDMR
++ rm /tmp/tmp.GzsvsEs3W2 /tmp/tmp.coAT1nJDMR
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.MqI6T8zsOe
++ mktemp
+ local LAST_ERR=/tmp/tmp.SjT9MIK0NI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MqI6T8zsOe
+ cat /tmp/tmp.SjT9MIK0NI
+ rm /tmp/tmp.MqI6T8zsOe /tmp/tmp.SjT9MIK0NI
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
some-name-rs0-0
+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
some-name-rs0-1
+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
some-name-rs0-2
+ desc 'add backups schedule, wait for the first backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
add backups schedule, wait for the first backup
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.5.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8"'
+ local LAST_OUT=/tmp/tmp.WrwlOxGaxD
++ mktemp
+ local LAST_ERR=/tmp/tmp.aS8118Nh6e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WrwlOxGaxD
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.aS8118Nh6e
+ rm /tmp/tmp.WrwlOxGaxD /tmp/tmp.aS8118Nh6e
+ return 0
+ sleep 55
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
++ mktemp
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.vL88M90lof
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.5.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8"'
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ local LAST_ERR=/tmp/tmp.cReDemBpU7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vL88M90lof
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.cReDemBpU7
+ rm /tmp/tmp.vL88M90lof /tmp/tmp.cReDemBpU7
+ return 0
+ '[' -z '' ']'
++ kubectl_bin get psmdb-backup
++ awk '{print$1}'
++ grep aws-s3
+++ mktemp
++ head -1
++ local LAST_OUT=/tmp/tmp.kCZqpEdPXn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AjuUh0ZcvK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kCZqpEdPXn
++ cat /tmp/tmp.AjuUh0ZcvK
++ rm /tmp/tmp.kCZqpEdPXn /tmp/tmp.AjuUh0ZcvK
++ return 0
+ backup_name_aws=cron-some-name-20240912233400-jgdjd
++ kubectl_bin get psmdb-backup
++ grep gcp-cs
+++ mktemp
++ awk '{print$1}'
++ head -1
++ local LAST_OUT=/tmp/tmp.e8gwL5nGbK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.01GWjOZuUK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.e8gwL5nGbK
++ cat /tmp/tmp.01GWjOZuUK
++ rm /tmp/tmp.e8gwL5nGbK /tmp/tmp.01GWjOZuUK
++ return 0
+ backup_name_gcp=cron-some-name-20240912233400-w96g6
++ grep azure-blob
++ kubectl_bin get psmdb-backup
++ head -1
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vVrWD7hjMW
++ awk '{print$1}'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.18AXVbp8L2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vVrWD7hjMW
++ cat /tmp/tmp.18AXVbp8L2
++ rm /tmp/tmp.vVrWD7hjMW /tmp/tmp.18AXVbp8L2
++ return 0
+ backup_name_azure=cron-some-name-20240912233400-ltwtp
+ wait_backup cron-some-name-20240912233400-jgdjd
+ local backup_name=cron-some-name-20240912233400-jgdjd
+ set +o xtrace
cron-some-name-20240912233400-jgdjd..........................
+ wait_backup cron-some-name-20240912233400-w96g6
+ local backup_name=cron-some-name-20240912233400-w96g6
+ set +o xtrace
cron-some-name-20240912233400-w96g6.
+ wait_backup cron-some-name-20240912233400-ltwtp
+ local backup_name=cron-some-name-20240912233400-ltwtp
+ set +o xtrace
cron-some-name-20240912233400-ltwtp.
++ kubectl_bin get psmdb-backup
++ grep minio
+++ mktemp
++ awk '{print$1}'
++ local LAST_OUT=/tmp/tmp.KZWTRY90tR
++ head -1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.abAxFvJ2KY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KZWTRY90tR
++ cat /tmp/tmp.abAxFvJ2KY
++ rm /tmp/tmp.KZWTRY90tR /tmp/tmp.abAxFvJ2KY
++ return 0
+ backup_name_minio=cron-some-name-20240912233400-2x45r
+ wait_backup cron-some-name-20240912233400-2x45r
+ local backup_name=cron-some-name-20240912233400-2x45r
+ set +o xtrace
cron-some-name-20240912233400-2x45r.............
+ sleep 5
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20240912233400-2x45r
++ local backup_name=cron-some-name-20240912233400-2x45r
++ sed 's|s3://||'
++ kubectl_bin get psmdb-backup cron-some-name-20240912233400-2x45r -o 'jsonpath={.status.destination}'
+++ mktemp
++ sed -e 's/.json$//'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.Ic0xzM0GIX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AGnDjQXFRq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20240912233400-2x45r -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ic0xzM0GIX
++ cat /tmp/tmp.AGnDjQXFRq
++ rm /tmp/tmp.Ic0xzM0GIX /tmp/tmp.AGnDjQXFRq
++ return 0
+ backup_dest_minio=operator-testing/2024-09-12T23:35:55Z
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2024-09-12T23:35:55Z/rs0/
+ grep myApp.test.gz
++ mktemp
+ local LAST_OUT=/tmp/tmp.6MvkJo8z2S
++ mktemp
+ local LAST_ERR=/tmp/tmp.yBkvmIOa0q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2024-09-12T23:35:55Z/rs0/
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6MvkJo8z2S
+ cat /tmp/tmp.yBkvmIOa0q
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-11064
+ rm /tmp/tmp.6MvkJo8z2S /tmp/tmp.yBkvmIOa0q
+ return 0
2024-09-12 23:36:00         55 myApp.test.gz
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.11DyOGZqm8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tDlyoVvWmG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.11DyOGZqm8
++ cat /tmp/tmp.tDlyoVvWmG
++ rm /tmp/tmp.11DyOGZqm8 /tmp/tmp.tDlyoVvWmG
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.eLBQNTqGIf
++ mktemp
+ local LAST_ERR=/tmp/tmp.VkrtanxsuR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.eLBQNTqGIf
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c968383f-ef57-40b1-a9b5-0432c45ba7ce") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.VkrtanxsuR
+ rm /tmp/tmp.eLBQNTqGIf /tmp/tmp.VkrtanxsuR
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cgKLtz8Ebn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K7BK9XifRh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cgKLtz8Ebn
++ cat /tmp/tmp.K7BK9XifRh
++ rm /tmp/tmp.cgKLtz8Ebn /tmp/tmp.K7BK9XifRh
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.V1ANNyWJKp
++ mktemp
+ local LAST_ERR=/tmp/tmp.qvKpZXfnqC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V1ANNyWJKp
+ cat /tmp/tmp.qvKpZXfnqC
+ rm /tmp/tmp.V1ANNyWJKp /tmp/tmp.qvKpZXfnqC
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.38t13XVDHI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XvDg67mFNq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.38t13XVDHI
++ cat /tmp/tmp.XvDg67mFNq
++ rm /tmp/tmp.38t13XVDHI /tmp/tmp.XvDg67mFNq
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nM4cX0SM1t
++ mktemp
+ local LAST_ERR=/tmp/tmp.WrezLQQq5A
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nM4cX0SM1t
+ cat /tmp/tmp.WrezLQQq5A
+ rm /tmp/tmp.nM4cX0SM1t /tmp/tmp.WrezLQQq5A
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.IO3hc1VqUL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SeNFi3TTjr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IO3hc1VqUL
++ cat /tmp/tmp.SeNFi3TTjr
++ rm /tmp/tmp.IO3hc1VqUL /tmp/tmp.SeNFi3TTjr
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.A1IKC3fYLL
++ mktemp
+ local LAST_ERR=/tmp/tmp.UESkm7z91x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.A1IKC3fYLL
+ cat /tmp/tmp.UESkm7z91x
+ rm /tmp/tmp.A1IKC3fYLL /tmp/tmp.UESkm7z91x
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ run_restore cron-some-name-20240912233400-2x45r
+ local backup_name=cron-some-name-20240912233400-2x45r
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/restore.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20240912233400-2x45r/'
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20240912233400-2x45r/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LFY36WotLr
++ mktemp
+ local LAST_ERR=/tmp/tmp.YWfenyIMDP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LFY36WotLr
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20240912233400-2x45r created
+ cat /tmp/tmp.YWfenyIMDP
+ rm /tmp/tmp.LFY36WotLr /tmp/tmp.YWfenyIMDP
+ return 0
+ wait_restore cron-some-name-20240912233400-2x45r some-name
+ local backup_name=cron-some-name-20240912233400-2x45r
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/cron-some-name-20240912233400-2x45r to reach ready state...........
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Do5t3fWtu8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZgwJY5sCFR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Do5t3fWtu8
++ cat /tmp/tmp.ZgwJY5sCFR
++ rm /tmp/tmp.Do5t3fWtu8 /tmp/tmp.ZgwJY5sCFR
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.IQlmt6mgR8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rqJEbokuvs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IQlmt6mgR8
++ cat /tmp/tmp.rqJEbokuvs
++ rm /tmp/tmp.IQlmt6mgR8 /tmp/tmp.rqJEbokuvs
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.IhYlmNwGnn
++ mktemp
+ local LAST_ERR=/tmp/tmp.v87oCFyAlK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.IhYlmNwGnn
+ cat /tmp/tmp.v87oCFyAlK
+ rm /tmp/tmp.IhYlmNwGnn /tmp/tmp.v87oCFyAlK
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5J5OOpvbll
+++ mktemp
++ local LAST_ERR=/tmp/tmp.00AqqhmQFx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5J5OOpvbll
++ cat /tmp/tmp.00AqqhmQFx
++ rm /tmp/tmp.5J5OOpvbll /tmp/tmp.00AqqhmQFx
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NCDbZ2t9ax
++ mktemp
+ local LAST_ERR=/tmp/tmp.oY0PFZRbtq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NCDbZ2t9ax
+ cat /tmp/tmp.oY0PFZRbtq
+ rm /tmp/tmp.NCDbZ2t9ax /tmp/tmp.oY0PFZRbtq
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yqi6OsXEQC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cAv63nDLTM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yqi6OsXEQC
++ cat /tmp/tmp.cAv63nDLTM
++ rm /tmp/tmp.yqi6OsXEQC /tmp/tmp.cAv63nDLTM
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.6GyyqIMsrO
++ mktemp
+ local LAST_ERR=/tmp/tmp.sIfv8U6GIm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6GyyqIMsrO
+ cat /tmp/tmp.sIfv8U6GIm
+ rm /tmp/tmp.6GyyqIMsrO /tmp/tmp.sIfv8U6GIm
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20240912233400-jgdjd
++ local backup_name=cron-some-name-20240912233400-jgdjd
++ kubectl_bin get psmdb-backup cron-some-name-20240912233400-jgdjd -o 'jsonpath={.status.destination}'
+++ mktemp
++ sed 's|s3://||'
++ sed 's|azure://||'
++ sed -e 's/.json$//'
++ local LAST_OUT=/tmp/tmp.33zueFH6SL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Psdom1g0f
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20240912233400-jgdjd -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.33zueFH6SL
++ cat /tmp/tmp.5Psdom1g0f
++ rm /tmp/tmp.33zueFH6SL /tmp/tmp.5Psdom1g0f
++ return 0
+ backup_dest_aws=operator-testing/psmdb-scheduled-backup/2024-09-12T23:35:24Z
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-scheduled-backup/2024-09-12T23:35:24Z/rs0/myApp.test.gz
+ gunzip
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t2ryxqS6Em
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IySSIRx8E2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t2ryxqS6Em
++ cat /tmp/tmp.IySSIRx8E2
++ rm /tmp/tmp.t2ryxqS6Em /tmp/tmp.IySSIRx8E2
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.8GTB7QsbjU
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bh3eHzw6qi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8GTB7QsbjU
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("4a0292ac-9bd4-44d3-b104-bc9ac6e1e9c2") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Bh3eHzw6qi
+ rm /tmp/tmp.8GTB7QsbjU /tmp/tmp.Bh3eHzw6qi
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.sefNEONQoA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5PJTZhuxHb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sefNEONQoA
++ cat /tmp/tmp.5PJTZhuxHb
++ rm /tmp/tmp.sefNEONQoA /tmp/tmp.5PJTZhuxHb
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.hzmCIU4Njx
++ mktemp
+ local LAST_ERR=/tmp/tmp.kVfil8N42o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hzmCIU4Njx
+ cat /tmp/tmp.kVfil8N42o
+ rm /tmp/tmp.hzmCIU4Njx /tmp/tmp.kVfil8N42o
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pHiXgtImfv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fvz51MpxiO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pHiXgtImfv
++ cat /tmp/tmp.fvz51MpxiO
++ rm /tmp/tmp.pHiXgtImfv /tmp/tmp.fvz51MpxiO
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zdczjPveUI
++ mktemp
+ local LAST_ERR=/tmp/tmp.2xchsm4Ujc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zdczjPveUI
+ cat /tmp/tmp.2xchsm4Ujc
+ rm /tmp/tmp.zdczjPveUI /tmp/tmp.2xchsm4Ujc
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UK9PFgOhqu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UdKITqzpYa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UK9PFgOhqu
++ cat /tmp/tmp.UdKITqzpYa
++ rm /tmp/tmp.UK9PFgOhqu /tmp/tmp.UdKITqzpYa
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.vPzo3H1rBW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Iw8RQuEmx5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vPzo3H1rBW
+ cat /tmp/tmp.Iw8RQuEmx5
+ rm /tmp/tmp.vPzo3H1rBW /tmp/tmp.Iw8RQuEmx5
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ run_restore cron-some-name-20240912233400-jgdjd
+ local backup_name=cron-some-name-20240912233400-jgdjd
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YGWkl7bymh
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20240912233400-jgdjd/'
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bh1RI4gx7m
+ local exit_status=0
+ local timeout=4
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20240912233400-jgdjd/'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YGWkl7bymh
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20240912233400-jgdjd created
+ cat /tmp/tmp.Bh1RI4gx7m
+ rm /tmp/tmp.YGWkl7bymh /tmp/tmp.Bh1RI4gx7m
+ return 0
+ wait_restore cron-some-name-20240912233400-jgdjd some-name
+ local backup_name=cron-some-name-20240912233400-jgdjd
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/cron-some-name-20240912233400-jgdjd to reach ready state.....................
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SMTD2hm339
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SRhsubZBIy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SMTD2hm339
++ cat /tmp/tmp.SRhsubZBIy
++ rm /tmp/tmp.SMTD2hm339 /tmp/tmp.SRhsubZBIy
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OJnhEFlsWH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8Mc920ABdG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OJnhEFlsWH
++ cat /tmp/tmp.8Mc920ABdG
++ rm /tmp/tmp.OJnhEFlsWH /tmp/tmp.8Mc920ABdG
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iJwWwqr6Vy
++ mktemp
+ local LAST_ERR=/tmp/tmp.IfPHtGhJqj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iJwWwqr6Vy
+ cat /tmp/tmp.IfPHtGhJqj
+ rm /tmp/tmp.iJwWwqr6Vy /tmp/tmp.IfPHtGhJqj
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Swn34B3WFO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LUAt30ULhP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Swn34B3WFO
++ cat /tmp/tmp.LUAt30ULhP
++ rm /tmp/tmp.Swn34B3WFO /tmp/tmp.LUAt30ULhP
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.H9TdOzqwE3
++ mktemp
+ local LAST_ERR=/tmp/tmp.wzn3K8nyVK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.H9TdOzqwE3
+ cat /tmp/tmp.wzn3K8nyVK
+ rm /tmp/tmp.H9TdOzqwE3 /tmp/tmp.wzn3K8nyVK
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nka75wjdeB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ksnAu8Kngo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nka75wjdeB
++ cat /tmp/tmp.ksnAu8Kngo
++ rm /tmp/tmp.Nka75wjdeB /tmp/tmp.ksnAu8Kngo
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZdCsFUp97N
++ mktemp
+ local LAST_ERR=/tmp/tmp.8vDnBYLyGR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZdCsFUp97N
+ cat /tmp/tmp.8vDnBYLyGR
+ rm /tmp/tmp.ZdCsFUp97N /tmp/tmp.8vDnBYLyGR
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20240912233400-w96g6
++ local backup_name=cron-some-name-20240912233400-w96g6
++ kubectl_bin get psmdb-backup cron-some-name-20240912233400-w96g6 -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Bbfar5sSoU
+++ mktemp
++ sed -e 's/.json$//'
++ local LAST_ERR=/tmp/tmp.0nH7jcfp4F
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20240912233400-w96g6 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Bbfar5sSoU
++ cat /tmp/tmp.0nH7jcfp4F
++ rm /tmp/tmp.Bbfar5sSoU /tmp/tmp.0nH7jcfp4F
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-scheduled-backup/2024-09-12T23:34:53Z
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-scheduled-backup/2024-09-12T23:34:53Z/rs0/myApp.test.gz
+ gunzip
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3I6VB2Hcou
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ulUiI8AyWF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3I6VB2Hcou
++ cat /tmp/tmp.ulUiI8AyWF
++ rm /tmp/tmp.3I6VB2Hcou /tmp/tmp.ulUiI8AyWF
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.rfo4LMJzXi
++ mktemp
+ local LAST_ERR=/tmp/tmp.EZKJUdO8rC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.rfo4LMJzXi
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("5ae890fb-6a18-45e0-a261-a8e8b3545b18") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.EZKJUdO8rC
+ rm /tmp/tmp.rfo4LMJzXi /tmp/tmp.EZKJUdO8rC
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iMdHcehXsT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1udj3ZiX6U
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iMdHcehXsT
++ cat /tmp/tmp.1udj3ZiX6U
++ rm /tmp/tmp.iMdHcehXsT /tmp/tmp.1udj3ZiX6U
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lFU2tI4lyH
++ mktemp
+ local LAST_ERR=/tmp/tmp.nZJubHjAKx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lFU2tI4lyH
+ cat /tmp/tmp.nZJubHjAKx
+ rm /tmp/tmp.lFU2tI4lyH /tmp/tmp.nZJubHjAKx
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TkIL8lbjdk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AK40u720XK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TkIL8lbjdk
++ cat /tmp/tmp.AK40u720XK
++ rm /tmp/tmp.TkIL8lbjdk /tmp/tmp.AK40u720XK
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0YRlp3yMgi
++ mktemp
+ local LAST_ERR=/tmp/tmp.7N0EKJ2CQ4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0YRlp3yMgi
+ cat /tmp/tmp.7N0EKJ2CQ4
+ rm /tmp/tmp.0YRlp3yMgi /tmp/tmp.7N0EKJ2CQ4
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z1ICmX2gJl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.57Te0RVaOm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.z1ICmX2gJl
++ cat /tmp/tmp.57Te0RVaOm
++ rm /tmp/tmp.z1ICmX2gJl /tmp/tmp.57Te0RVaOm
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.7dDa2GbDF3
++ mktemp
+ local LAST_ERR=/tmp/tmp.cnqPl3VtgB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7dDa2GbDF3
+ cat /tmp/tmp.cnqPl3VtgB
+ rm /tmp/tmp.7dDa2GbDF3 /tmp/tmp.cnqPl3VtgB
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ run_restore cron-some-name-20240912233400-w96g6
+ local backup_name=cron-some-name-20240912233400-w96g6
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20240912233400-w96g6/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.au1qvYMpdv
++ mktemp
+ local LAST_ERR=/tmp/tmp.ig0DEBUTiB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20240912233400-w96g6/'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.au1qvYMpdv
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20240912233400-w96g6 created
+ cat /tmp/tmp.ig0DEBUTiB
+ rm /tmp/tmp.au1qvYMpdv /tmp/tmp.ig0DEBUTiB
+ return 0
+ wait_restore cron-some-name-20240912233400-w96g6 some-name
+ local backup_name=cron-some-name-20240912233400-w96g6
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/cron-some-name-20240912233400-w96g6 to reach ready state..................
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tgquAm7x42
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2XyXsb3f3P
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tgquAm7x42
++ cat /tmp/tmp.2XyXsb3f3P
++ rm /tmp/tmp.tgquAm7x42 /tmp/tmp.2XyXsb3f3P
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.scTi8O2BcX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.k8OlXUIpmL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.scTi8O2BcX
++ cat /tmp/tmp.k8OlXUIpmL
++ rm /tmp/tmp.scTi8O2BcX /tmp/tmp.k8OlXUIpmL
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pYBdelMfU3
++ mktemp
+ local LAST_ERR=/tmp/tmp.J4eLmR862a
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pYBdelMfU3
+ cat /tmp/tmp.J4eLmR862a
+ rm /tmp/tmp.pYBdelMfU3 /tmp/tmp.J4eLmR862a
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MY7guNPKmm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IdxrYqTNZw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MY7guNPKmm
++ cat /tmp/tmp.IdxrYqTNZw
++ rm /tmp/tmp.MY7guNPKmm /tmp/tmp.IdxrYqTNZw
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3jDNWMdPGG
++ mktemp
+ local LAST_ERR=/tmp/tmp.7i7BkLd54Z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3jDNWMdPGG
+ cat /tmp/tmp.7i7BkLd54Z
+ rm /tmp/tmp.3jDNWMdPGG /tmp/tmp.7i7BkLd54Z
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.B8H3d6zTjG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pjOtqURI6Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.B8H3d6zTjG
++ cat /tmp/tmp.pjOtqURI6Z
++ rm /tmp/tmp.B8H3d6zTjG /tmp/tmp.pjOtqURI6Z
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yGMN10hf19
++ mktemp
+ local LAST_ERR=/tmp/tmp.OEpTMCB4Yf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yGMN10hf19
+ cat /tmp/tmp.OEpTMCB4Yf
+ rm /tmp/tmp.yGMN10hf19 /tmp/tmp.OEpTMCB4Yf
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20240912233400-ltwtp
++ local backup_name=cron-some-name-20240912233400-ltwtp
++ kubectl_bin get psmdb-backup cron-some-name-20240912233400-ltwtp -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ErwdNlttKg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y1rIUQGQFF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20240912233400-ltwtp -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ErwdNlttKg
++ cat /tmp/tmp.y1rIUQGQFF
++ rm /tmp/tmp.ErwdNlttKg /tmp/tmp.y1rIUQGQFF
++ return 0
+ backup_dest_azure=operator-testing/psmdb-scheduled-backup/2024-09-12T23:34:21Z
+ gunzip
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-scheduled-backup/2024-09-12T23:34:21Z/rs0/myApp.test.gz
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-11064
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bWKTAoVw5v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Jjs1rzdXmg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bWKTAoVw5v
++ cat /tmp/tmp.Jjs1rzdXmg
++ rm /tmp/tmp.bWKTAoVw5v /tmp/tmp.Jjs1rzdXmg
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ng25a0RKVa
++ mktemp
+ local LAST_ERR=/tmp/tmp.O26p6jSfDI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ng25a0RKVa
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("65002671-535e-4b03-bd83-20dcb248e2ea") }
Percona Server for MongoDB server version: v7.0.12-7
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.O26p6jSfDI
+ rm /tmp/tmp.ng25a0RKVa /tmp/tmp.O26p6jSfDI
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0HYaBnPInL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8LFJsndzaL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0HYaBnPInL
++ cat /tmp/tmp.8LFJsndzaL
++ rm /tmp/tmp.0HYaBnPInL /tmp/tmp.8LFJsndzaL
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pL0f9DdiU6
++ mktemp
+ local LAST_ERR=/tmp/tmp.6FMl7nfv5v
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pL0f9DdiU6
+ cat /tmp/tmp.6FMl7nfv5v
+ rm /tmp/tmp.pL0f9DdiU6 /tmp/tmp.6FMl7nfv5v
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oxdpgeFUHQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hkdStrEVMK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oxdpgeFUHQ
++ cat /tmp/tmp.hkdStrEVMK
++ rm /tmp/tmp.oxdpgeFUHQ /tmp/tmp.hkdStrEVMK
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.jsUbzZVqXS
++ mktemp
+ local LAST_ERR=/tmp/tmp.DB3TtLqpsD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jsUbzZVqXS
+ cat /tmp/tmp.DB3TtLqpsD
+ rm /tmp/tmp.jsUbzZVqXS /tmp/tmp.DB3TtLqpsD
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0Bn0WPdXLV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hgHIrLlBTf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0Bn0WPdXLV
++ cat /tmp/tmp.hgHIrLlBTf
++ rm /tmp/tmp.0Bn0WPdXLV /tmp/tmp.hgHIrLlBTf
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.C9LQs1WTSR
++ mktemp
+ local LAST_ERR=/tmp/tmp.MQUPU1dENl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C9LQs1WTSR
+ cat /tmp/tmp.MQUPU1dENl
+ rm /tmp/tmp.C9LQs1WTSR /tmp/tmp.MQUPU1dENl
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.ZIdgDFXkGo/find-2nd
+ run_restore cron-some-name-20240912233400-ltwtp
+ local backup_name=cron-some-name-20240912233400-ltwtp
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20240912233400-ltwtp/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20240912233400-ltwtp/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qdOfky2840
++ mktemp
+ local LAST_ERR=/tmp/tmp.xHoetvmeBu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qdOfky2840
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20240912233400-ltwtp created
+ cat /tmp/tmp.xHoetvmeBu
+ rm /tmp/tmp.qdOfky2840 /tmp/tmp.xHoetvmeBu
+ return 0
+ wait_restore cron-some-name-20240912233400-ltwtp some-name
+ local backup_name=cron-some-name-20240912233400-ltwtp
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/cron-some-name-20240912233400-ltwtp to reach ready state.................
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HmI2Kvnc6q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m2ZeY3vsE7
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HmI2Kvnc6q
++ cat /tmp/tmp.m2ZeY3vsE7
++ rm /tmp/tmp.HmI2Kvnc6q /tmp/tmp.m2ZeY3vsE7
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vvCdZDloaK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.avqLisqNRQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vvCdZDloaK
++ cat /tmp/tmp.avqLisqNRQ
++ rm /tmp/tmp.vvCdZDloaK /tmp/tmp.avqLisqNRQ
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HL0S2VTfjz
++ mktemp
+ local LAST_ERR=/tmp/tmp.S1IJp6nQcL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HL0S2VTfjz
+ cat /tmp/tmp.S1IJp6nQcL
+ rm /tmp/tmp.HL0S2VTfjz /tmp/tmp.S1IJp6nQcL
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TGIkZlrU8T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UUGWeWH4XJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TGIkZlrU8T
++ cat /tmp/tmp.UUGWeWH4XJ
++ rm /tmp/tmp.TGIkZlrU8T /tmp/tmp.UUGWeWH4XJ
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.l8sTI5Ho4d
++ mktemp
+ local LAST_ERR=/tmp/tmp.QTx4KMhReO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.l8sTI5Ho4d
+ cat /tmp/tmp.QTx4KMhReO
+ rm /tmp/tmp.l8sTI5Ho4d /tmp/tmp.QTx4KMhReO
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.0Bl1myUa3s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yUgROv2lYJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0Bl1myUa3s
++ cat /tmp/tmp.yUgROv2lYJ
++ rm /tmp/tmp.0Bl1myUa3s /tmp/tmp.yUgROv2lYJ
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.USIJ5oXg38
++ mktemp
+ local LAST_ERR=/tmp/tmp.hdldSlQNP2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.USIJ5oXg38
+ cat /tmp/tmp.hdldSlQNP2
+ rm /tmp/tmp.USIJ5oXg38 /tmp/tmp.hdldSlQNP2
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ desc 'add physical backup schedule, wait for the first backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
add physical backup schedule, wait for the first backup
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
++ mktemp
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ local LAST_OUT=/tmp/tmp.21oHOfrhvl
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.5.0"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.43dlTwlLc5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.21oHOfrhvl
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.43dlTwlLc5
+ rm /tmp/tmp.21oHOfrhvl /tmp/tmp.43dlTwlLc5
+ return 0
+ sleep 55
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.NB1OpC3RS1
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "percona/percona-backup-mongodb:2.5.0"'
+ local LAST_ERR=/tmp/tmp.m5a98ZrcE0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1641-df7592c8"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NB1OpC3RS1
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.m5a98ZrcE0
+ rm /tmp/tmp.NB1OpC3RS1 /tmp/tmp.m5a98ZrcE0
+ return 0
++ tail -1
++ kubectl_bin get psmdb-backup
++ awk '{print $1}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sp1lHINIKy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TmnUpk7Uf5
++ local exit_status=0
++ local timeout=4
++ grep minio
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.sp1lHINIKy
++ cat /tmp/tmp.TmnUpk7Uf5
++ rm /tmp/tmp.sp1lHINIKy /tmp/tmp.TmnUpk7Uf5
++ return 0
+ backup_name_minio=cron-some-name-20240912233400-2x45r
+ wait_backup cron-some-name-20240912233400-2x45r
+ local backup_name=cron-some-name-20240912233400-2x45r
+ set +o xtrace
cron-some-name-20240912233400-2x45r.
+ sleep 5
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ run_restore cron-some-name-20240912233400-2x45r 3
+ local backup_name=cron-some-name-20240912233400-2x45r
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20240912233400-2x45r/'
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20240912233400-2x45r/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Draf0ZNQAQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.3VpAvbxeAj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Draf0ZNQAQ
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20240912233400-2x45r unchanged
+ cat /tmp/tmp.3VpAvbxeAj
+ rm /tmp/tmp.Draf0ZNQAQ /tmp/tmp.3VpAvbxeAj
+ return 0
+ wait_restore cron-some-name-20240912233400-2x45r some-name
+ local backup_name=cron-some-name-20240912233400-2x45r
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=780
+ set +o xtrace
waiting psmdb-restore/cron-some-name-20240912233400-2x45r to reach ready state.
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3K6FWNB4cP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NYmMLjcU9I
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3K6FWNB4cP
++ cat /tmp/tmp.NYmMLjcU9I
++ rm /tmp/tmp.3K6FWNB4cP /tmp/tmp.NYmMLjcU9I
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TNDuYny6ci
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ypHGtVk25i
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TNDuYny6ci
++ cat /tmp/tmp.ypHGtVk25i
++ rm /tmp/tmp.TNDuYny6ci /tmp/tmp.ypHGtVk25i
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.LHFK9BhIYU
++ mktemp
+ local LAST_ERR=/tmp/tmp.kge9zknASp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LHFK9BhIYU
+ cat /tmp/tmp.kge9zknASp
+ rm /tmp/tmp.LHFK9BhIYU /tmp/tmp.kge9zknASp
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 mongodb ''
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.RngWiIEfPW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BDxqwmLEzG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RngWiIEfPW
++ cat /tmp/tmp.BDxqwmLEzG
++ rm /tmp/tmp.RngWiIEfPW /tmp/tmp.BDxqwmLEzG
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BNaPjtn39W
++ mktemp
+ local LAST_ERR=/tmp/tmp.2IytIcjnll
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BNaPjtn39W
+ cat /tmp/tmp.2IytIcjnll
+ rm /tmp/tmp.BNaPjtn39W /tmp/tmp.2IytIcjnll
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LeTNtMDAiL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9M0SAsvreK
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LeTNtMDAiL
++ cat /tmp/tmp.9M0SAsvreK
++ rm /tmp/tmp.LeTNtMDAiL /tmp/tmp.9M0SAsvreK
++ return 0
+ local client_container=psmdb-client-6c585f8dbd-sdtv4
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.zRE7to1IEn
++ mktemp
+ local LAST_ERR=/tmp/tmp.yYvFDJ9tk1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-6c585f8dbd-sdtv4 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-11064.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zRE7to1IEn
+ cat /tmp/tmp.yYvFDJ9tk1
+ rm /tmp/tmp.zRE7to1IEn /tmp/tmp.yYvFDJ9tk1
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.ZIdgDFXkGo/find
+ sleep 60
+ unlabel_node
+ desc 'remove labels'
+ set +o xtrace


-----------------------------------------------------------------------------------
remove labels
-----------------------------------------------------------------------------------

+ kubectl_bin label nodes gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 backupWorker- --overwrite
++ mktemp
+ local LAST_OUT=/tmp/tmp.YEthLCNRLP
++ mktemp
+ local LAST_ERR=/tmp/tmp.MCGNKhSWzl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label nodes gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 backupWorker- --overwrite
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YEthLCNRLP
node/gke-jen-psmdb-1641-df759-default-pool-a1d895a0-2x09 labeled
+ cat /tmp/tmp.MCGNKhSWzl
+ rm /tmp/tmp.YEthLCNRLP /tmp/tmp.MCGNKhSWzl
+ return 0
+ destroy scheduled-backup-11064
+ local namespace=scheduled-backup-11064
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.hzi6kRUvrB
++ mktemp
+ local LAST_ERR=/tmp/tmp.7SAPkZ9kh3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hzi6kRUvrB
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.7SAPkZ9kh3
+ rm /tmp/tmp.hzi6kRUvrB /tmp/tmp.7SAPkZ9kh3
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-11064 cron-some-name-20240912233400-2x45r --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912233400-2x45r patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-11064 cron-some-name-20240912233400-jgdjd --type=merge -p '{"metadata":{"finalizers":[]}}'
E0912 23:43:42.686562   31276 memcache.go:287] "Unhandled Error" err="couldn't get resource list for psmdb.percona.com/v1-10-0: the server could not find the requested resource"
E0912 23:43:42.713946   31276 memcache.go:287] "Unhandled Error" err="couldn't get resource list for psmdb.percona.com/v1-11-0: the server could not find the requested resource"
E0912 23:43:42.715605   31276 memcache.go:287] "Unhandled Error" err="couldn't get resource list for psmdb.percona.com/v1-12-0: the server could not find the requested resource"
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912233400-jgdjd patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-11064 cron-some-name-20240912233400-ltwtp --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912233400-ltwtp patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-11064 cron-some-name-20240912233400-w96g6 --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20240912233400-w96g6 patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.N80Up4a0Gn
++ mktemp
+ local LAST_ERR=/tmp/tmp.4VH2ylAnfV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.N80Up4a0Gn
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.4VH2ylAnfV
+ rm /tmp/tmp.N80Up4a0Gn /tmp/tmp.4VH2ylAnfV
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.2lqveqIT8m
++ mktemp
+ local LAST_ERR=/tmp/tmp.1M0fu7Cy2D
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2lqveqIT8m
+ cat /tmp/tmp.1M0fu7Cy2D
+ rm /tmp/tmp.2lqveqIT8m /tmp/tmp.1M0fu7Cy2D
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.B19tqJC2Rl
++ mktemp
+ local LAST_ERR=/tmp/tmp.ivl805MDlq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B19tqJC2Rl
+ cat /tmp/tmp.ivl805MDlq
+ rm /tmp/tmp.B19tqJC2Rl /tmp/tmp.ivl805MDlq
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.b9m1UgNhG3
++ mktemp
+ local LAST_ERR=/tmp/tmp.DOM8FpbUVl
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1641/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b9m1UgNhG3
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.DOM8FpbUVl
+ rm /tmp/tmp.b9m1UgNhG3 /tmp/tmp.DOM8FpbUVl
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.x8Qe91XIYw
++ mktemp
+ local LAST_ERR=/tmp/tmp.p4D09mzykW
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.x8Qe91XIYw
+ cat /tmp/tmp.p4D09mzykW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.x8Qe91XIYw
+ cat /tmp/tmp.p4D09mzykW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.x8Qe91XIYw
+ cat /tmp/tmp.p4D09mzykW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.x8Qe91XIYw
+ cat /tmp/tmp.p4D09mzykW
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.x8Qe91XIYw /tmp/tmp.p4D09mzykW
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.ZIdgDFXkGo
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ kubectl_bin delete --grace-period=0 --force=true namespace scheduled-backup-11064
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lec4OeuNWO
++ mktemp
+ local LAST_ERR=/tmp/tmp.cYsK8MhBAH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace scheduled-backup-11064
++ mktemp
+ local LAST_OUT=/tmp/tmp.gysSpBfb7o
++ mktemp
+ local LAST_ERR=/tmp/tmp.twUSTHro6B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator