Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/logs/scheduled-backup.log
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.29) exceeds the supported minor version skew of +/-1
+ create_infra scheduled-backup-12587
+ local ns=scheduled-backup-12587
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.p5P0PMhUA3
++ mktemp
+ local LAST_ERR=/tmp/tmp.W9CgQEjYoP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p5P0PMhUA3
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.W9CgQEjYoP
+ rm /tmp/tmp.p5P0PMhUA3 /tmp/tmp.W9CgQEjYoP
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-14757 cron-some-name-20250324032100-94jcj --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20250324032100-94jcj patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-14757 cron-some-name-20250324032600-5p7sq --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/cron-some-name-20250324032600-5p7sq patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.xWR6Kskej6
++ mktemp
+ local LAST_ERR=/tmp/tmp.ycUIhB1nS6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xWR6Kskej6
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.ycUIhB1nS6
+ rm /tmp/tmp.xWR6Kskej6 /tmp/tmp.ycUIhB1nS6
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.5uG9COf9ws
++ mktemp
+ local LAST_ERR=/tmp/tmp.DZMSYIuMel
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5uG9COf9ws
+ cat /tmp/tmp.DZMSYIuMel
+ rm /tmp/tmp.5uG9COf9ws /tmp/tmp.DZMSYIuMel
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.D2HaAolbjq
++ mktemp
+ local LAST_ERR=/tmp/tmp.hKmGrGyLbq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.D2HaAolbjq
+ cat /tmp/tmp.hKmGrGyLbq
+ rm /tmp/tmp.D2HaAolbjq /tmp/tmp.hKmGrGyLbq
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.LdmdMYY1Nd
++ mktemp
+ local LAST_ERR=/tmp/tmp.vIA8caLMy2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LdmdMYY1Nd
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.vIA8caLMy2
+ rm /tmp/tmp.LdmdMYY1Nd /tmp/tmp.vIA8caLMy2
+ return 0
+ check_crd_for_deletion PR-1872-66b64516
+ local git_tag=PR-1872-66b64516
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1872-66b64516/deploy/crd.yaml
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UViE9M6jIy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8lqGrJLecn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.UViE9M6jIy
++ cat /tmp/tmp.8lqGrJLecn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.UViE9M6jIy
++ cat /tmp/tmp.8lqGrJLecn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.UViE9M6jIy
++ cat /tmp/tmp.8lqGrJLecn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.UViE9M6jIy
++ cat /tmp/tmp.8lqGrJLecn
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.UViE9M6jIy /tmp/tmp.8lqGrJLecn
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
+ xargs kubectl delete ns
+ kubectl_bin get ns
++ mktemp
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ local LAST_OUT=/tmp/tmp.z0W6Weesmj
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.dvpsN9RpOX
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.yvpJTU8dPb
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ local LAST_ERR=/tmp/tmp.3JS5MrEFs5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yvpJTU8dPb
+ cat /tmp/tmp.3JS5MrEFs5
+ rm /tmp/tmp.yvpJTU8dPb /tmp/tmp.3JS5MrEFs5
+ return 0
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "scheduled-backup-14757" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.z0W6Weesmj
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.dvpsN9RpOX
+ rm /tmp/tmp.z0W6Weesmj /tmp/tmp.dvpsN9RpOX
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.wcFwJhVgIb
++ mktemp
+ local LAST_ERR=/tmp/tmp.GFROLd7k0l
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wcFwJhVgIb
+ cat /tmp/tmp.GFROLd7k0l
+ rm /tmp/tmp.wcFwJhVgIb /tmp/tmp.GFROLd7k0l
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.gAeWaMxQIV
++ mktemp
+ local LAST_ERR=/tmp/tmp.cvDZ2xuSoO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gAeWaMxQIV
namespace/psmdb-operator created
+ cat /tmp/tmp.cvDZ2xuSoO
+ rm /tmp/tmp.gAeWaMxQIV /tmp/tmp.cvDZ2xuSoO
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.37KRoAKGKe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Z8h3dschjc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.37KRoAKGKe
++ cat /tmp/tmp.Z8h3dschjc
++ rm /tmp/tmp.37KRoAKGKe /tmp/tmp.Z8h3dschjc
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.8OZK3n5aMZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.6kHiZcSXqX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8OZK3n5aMZ
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1" modified.
+ cat /tmp/tmp.6kHiZcSXqX
+ rm /tmp/tmp.8OZK3n5aMZ /tmp/tmp.6kHiZcSXqX
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PIJeWKDg8p
++ mktemp
+ local LAST_ERR=/tmp/tmp.JlRnuH3mrG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PIJeWKDg8p
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.JlRnuH3mrG
+ rm /tmp/tmp.PIJeWKDg8p /tmp/tmp.JlRnuH3mrG
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YvyPurDvGC
++ mktemp
+ local LAST_ERR=/tmp/tmp.qR8N98glUv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YvyPurDvGC
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.qR8N98glUv
+ rm /tmp/tmp.YvyPurDvGC /tmp/tmp.qR8N98glUv
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.YdinutbkLr
++ mktemp
+ local LAST_ERR=/tmp/tmp.IYtmC9QKgT
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YdinutbkLr
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.IYtmC9QKgT
+ rm /tmp/tmp.YdinutbkLr /tmp/tmp.IYtmC9QKgT
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Px0A6WdaE9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TZ54mj1gYw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Px0A6WdaE9
++ cat /tmp/tmp.TZ54mj1gYw
++ rm /tmp/tmp.Px0A6WdaE9 /tmp/tmp.TZ54mj1gYw
++ return 0
+ wait_pod percona-server-mongodb-operator-559546695f-rr9bh
+ local pod=percona-server-mongodb-operator-559546695f-rr9bh
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-559546695f-rr9bh to be ready.OK
+ create_namespace scheduled-backup-12587
+ local namespace=scheduled-backup-12587
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces scheduled-backup-12587'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces scheduled-backup-12587
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace scheduled-backup-12587 --ignore-not-found
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NPcXQJdEES
+ xargs kubectl delete ns
++ mktemp
+ kubectl_bin get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.fDEnQP8Ean
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.MyYDFxplPM
++ seq 0 2
++ mktemp
+ local LAST_ERR=/tmp/tmp.3n2IxUi8wc
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scheduled-backup-12587 --ignore-not-found
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MyYDFxplPM
+ cat /tmp/tmp.3n2IxUi8wc
+ rm /tmp/tmp.MyYDFxplPM /tmp/tmp.3n2IxUi8wc
+ return 0
namespace "gke-managed-cim" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NPcXQJdEES
+ cat /tmp/tmp.fDEnQP8Ean
+ rm /tmp/tmp.NPcXQJdEES /tmp/tmp.fDEnQP8Ean
+ return 0
+ kubectl_bin wait --for=delete namespace scheduled-backup-12587
++ mktemp
+ local LAST_OUT=/tmp/tmp.20mEJUrQDZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.JNPJDGSW6T
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace scheduled-backup-12587
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.20mEJUrQDZ
+ cat /tmp/tmp.JNPJDGSW6T
+ rm /tmp/tmp.20mEJUrQDZ /tmp/tmp.JNPJDGSW6T
+ return 0
+ desc 'create namespace scheduled-backup-12587'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace scheduled-backup-12587
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace scheduled-backup-12587
++ mktemp
+ local LAST_OUT=/tmp/tmp.lCDvSWC6Hm
++ mktemp
+ local LAST_ERR=/tmp/tmp.7TylKmevaF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace scheduled-backup-12587
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lCDvSWC6Hm
namespace/scheduled-backup-12587 created
+ cat /tmp/tmp.7TylKmevaF
+ rm /tmp/tmp.lCDvSWC6Hm /tmp/tmp.7TylKmevaF
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Pr82RPcBf3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NGhzfVYgkR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Pr82RPcBf3
++ cat /tmp/tmp.NGhzfVYgkR
++ rm /tmp/tmp.Pr82RPcBf3 /tmp/tmp.NGhzfVYgkR
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1 --namespace=scheduled-backup-12587
++ mktemp
+ local LAST_OUT=/tmp/tmp.bS9U1IIHuL
++ mktemp
+ local LAST_ERR=/tmp/tmp.8Pf4IxhUnd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1 --namespace=scheduled-backup-12587
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.bS9U1IIHuL
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1872-66b64516-1-cluster1" modified.
+ cat /tmp/tmp.8Pf4IxhUnd
+ rm /tmp/tmp.bS9U1IIHuL /tmp/tmp.8Pf4IxhUnd
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Mar 24 03:28:56 2025
NAMESPACE: scheduled-backup-12587
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.scheduled-backup-12587.svc.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace scheduled-backup-12587 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace scheduled-backup-12587

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace scheduled-backup-12587 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace scheduled-backup-12587 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TolVlw5f3I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QV6ym7gJls
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TolVlw5f3I
++ cat /tmp/tmp.QV6ym7gJls
++ rm /tmp/tmp.TolVlw5f3I /tmp/tmp.QV6ym7gJls
++ return 0
+ MINIO_POD=minio-service-847fc8bb8d-wkpqk
+ wait_pod minio-service-847fc8bb8d-wkpqk
+ local pod=minio-service-847fc8bb8d-wkpqk
+ set +o xtrace
waiting for pod/minio-service-847fc8bb8d-wkpqk to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-12587.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dcy1Pu8tyX
++ mktemp
+ local LAST_ERR=/tmp/tmp.OcuuVObxWf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.scheduled-backup-12587.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Dcy1Pu8tyX
service/minio-service created
+ cat /tmp/tmp.OcuuVObxWf
+ rm /tmp/tmp.Dcy1Pu8tyX /tmp/tmp.OcuuVObxWf
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZkB7SexjwL
++ mktemp
+ local LAST_ERR=/tmp/tmp.7l3mYJ4jSb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZkB7SexjwL
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.7l3mYJ4jSb
+ rm /tmp/tmp.ZkB7SexjwL /tmp/tmp.7l3mYJ4jSb
+ return 0
+ label_node
++ kubectl_bin get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Of0YnPjC0y
+++ mktemp
++ head -n1
++ local LAST_ERR=/tmp/tmp.Jgo2tbTmgV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get nodes -o custom-columns=NAME:.metadata.name --no-headers=true
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Of0YnPjC0y
++ cat /tmp/tmp.Jgo2tbTmgV
++ rm /tmp/tmp.Of0YnPjC0y /tmp/tmp.Jgo2tbTmgV
++ return 0
+ LABELED_NODE=gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq
+ desc 'add labels'
+ set +o xtrace


-----------------------------------------------------------------------------------
add labels
-----------------------------------------------------------------------------------

+ kubectl_bin label nodes gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq backupWorker=True --overwrite
++ mktemp
+ local LAST_OUT=/tmp/tmp.5vQtJ0B1p9
++ mktemp
+ local LAST_ERR=/tmp/tmp.kyeDMpbAEp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label nodes gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq backupWorker=True --overwrite
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5vQtJ0B1p9
node/gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq not labeled
+ cat /tmp/tmp.kyeDMpbAEp
+ rm /tmp/tmp.5vQtJ0B1p9 /tmp/tmp.kyeDMpbAEp
+ return 0
+ desc 'create PriorityClass'
+ set +o xtrace


-----------------------------------------------------------------------------------
create PriorityClass
-----------------------------------------------------------------------------------

+ cat -
+ kubectl apply -f -
priorityclass.scheduling.k8s.io/high-priority configured
+ desc 'create secrets and start client'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets and start client
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Omu7hxwVGy
++ mktemp
+ local LAST_ERR=/tmp/tmp.iRKao7lizb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Omu7hxwVGy
secret/some-users created
deployment.apps/psmdb-client created
+ cat /tmp/tmp.iRKao7lizb
+ rm /tmp/tmp.Omu7hxwVGy /tmp/tmp.iRKao7lizb
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wjf0J87XzX
++ mktemp
+ local LAST_ERR=/tmp/tmp.7XEte2IfmX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Wjf0J87XzX
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.7XEte2IfmX
+ rm /tmp/tmp.Wjf0J87XzX /tmp/tmp.7XEte2IfmX
+ return 0
+ cluster=some-name-rs0
+ desc 'create first PSMDB cluster some-name-rs0'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PSMDB cluster some-name-rs0
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.02icoacaK3
++ mktemp
+ local LAST_ERR=/tmp/tmp.LAVsU7kvAw
+ local exit_status=0
+ local timeout=4
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ seq 0 2
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.02icoacaK3
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.LAVsU7kvAw
+ rm /tmp/tmp.02icoacaK3 /tmp/tmp.LAVsU7kvAw
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TyOKEHaX9D
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KZZGttgS5z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TyOKEHaX9D
++ cat /tmp/tmp.KZZGttgS5z
++ rm /tmp/tmp.TyOKEHaX9D /tmp/tmp.KZZGttgS5z
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.35yCHg5x1e
+++ mktemp
++ local LAST_ERR=/tmp/tmp.URIDvSir5o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.35yCHg5x1e
++ cat /tmp/tmp.URIDvSir5o
++ rm /tmp/tmp.35yCHg5x1e /tmp/tmp.URIDvSir5o
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness.
+ desc 'check if service and statefulset created with expected config'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if service and statefulset created with expected config
-----------------------------------------------------------------------------------

+ compare_kubectl statefulset/some-name-rs0
+ local resource=statefulset/some-name-rs0
+ local postfix=
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml
+ local new_result=/tmp/tmp.w55tYLOlFW/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.2ZXDJlZ0KM
++ mktemp
+ local LAST_ERR=/tmp/tmp.VOj6Sr7GOp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("scheduled-backup-12587", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2ZXDJlZ0KM
+ cat /tmp/tmp.VOj6Sr7GOp
+ rm /tmp/tmp.2ZXDJlZ0KM /tmp/tmp.VOj6Sr7GOp
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.w55tYLOlFW/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.29 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.w55tYLOlFW/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.w55tYLOlFW/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/statefulset_some-name-rs0.yml /tmp/tmp.w55tYLOlFW/statefulset_some-name-rs0.yml
+ desc 'create user myApp'
+ set +o xtrace


-----------------------------------------------------------------------------------
create user myApp
-----------------------------------------------------------------------------------

+ run_mongo 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-12587
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qOjeT9F6nB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jDHh1cYLE4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qOjeT9F6nB
++ cat /tmp/tmp.jDHh1cYLE4
++ rm /tmp/tmp.qOjeT9F6nB /tmp/tmp.jDHh1cYLE4
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.scoi9pf7mp
++ mktemp
+ local LAST_ERR=/tmp/tmp.Al6BdOIz7m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb+srv://userAdmin:userAdmin123456@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.scoi9pf7mp
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("cd4bf376-5efb-44cd-be2e-3499525da2aa") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.Al6BdOIz7m
+ rm /tmp/tmp.scoi9pf7mp /tmp/tmp.Al6BdOIz7m
+ return 0
+ sleep 2
+ desc 'write data, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data, read from all
-----------------------------------------------------------------------------------

+ run_mongo 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local 'command=use myApp\n db.test.insert({ x: 100500 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YqnFucnmC8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8KE8UM2thG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YqnFucnmC8
++ cat /tmp/tmp.8KE8UM2thG
++ rm /tmp/tmp.YqnFucnmC8 /tmp/tmp.8KE8UM2thG
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.tE181WYFVW
++ mktemp
+ local LAST_ERR=/tmp/tmp.Vlt41C3VVU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tE181WYFVW
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("d18197ba-380c-44da-9edc-2ee5945fef13") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Vlt41C3VVU
+ rm /tmp/tmp.tE181WYFVW /tmp/tmp.Vlt41C3VVU
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:31:22+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ly3lfIE8T5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AgKpeiffN0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ly3lfIE8T5
++ cat /tmp/tmp.AgKpeiffN0
++ rm /tmp/tmp.ly3lfIE8T5 /tmp/tmp.AgKpeiffN0
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mTGcy8psdY
++ mktemp
+ local LAST_ERR=/tmp/tmp.a8YdjBAREr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mTGcy8psdY
+ cat /tmp/tmp.a8YdjBAREr
+ rm /tmp/tmp.mTGcy8psdY /tmp/tmp.a8YdjBAREr
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:31:26+0000] running db.test.find() in myApp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DK3WxrnEkP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sheyRg9Ht0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DK3WxrnEkP
++ cat /tmp/tmp.sheyRg9Ht0
++ rm /tmp/tmp.DK3WxrnEkP /tmp/tmp.sheyRg9Ht0
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.wS44u356GP
++ mktemp
+ local LAST_ERR=/tmp/tmp.6lv3Z4xQiQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.wS44u356GP
+ cat /tmp/tmp.6lv3Z4xQiQ
+ rm /tmp/tmp.wS44u356GP /tmp/tmp.6lv3Z4xQiQ
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:31:30+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FTURjhsoBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.o5DrHBlDmz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FTURjhsoBI
++ cat /tmp/tmp.o5DrHBlDmz
++ rm /tmp/tmp.FTURjhsoBI /tmp/tmp.o5DrHBlDmz
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BXKis7W5N4
++ mktemp
+ local LAST_ERR=/tmp/tmp.haUltgnXsN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BXKis7W5N4
+ cat /tmp/tmp.haUltgnXsN
+ rm /tmp/tmp.BXKis7W5N4 /tmp/tmp.haUltgnXsN
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ wait_backup_agent some-name-rs0-0
+ local agent_pod=some-name-rs0-0
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-0...2025-03-24T03:31:05.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-1
+ local agent_pod=some-name-rs0-1
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-1...2025-03-24T03:31:08.000+0000 I listening for the commands

+ wait_backup_agent some-name-rs0-2
+ local agent_pod=some-name-rs0-2
+ set +o xtrace
waiting for pbm-agent to be ready in some-name-rs0-2...2025-03-24T03:31:09.000+0000 I listening for the commands

+ desc 'add backups schedule, wait for backups'
+ set +o xtrace


-----------------------------------------------------------------------------------
add backups schedule, wait for backups
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.J0qV3JQ9jP
++ mktemp
+ local LAST_ERR=/tmp/tmp.s5pQsOIPqI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J0qV3JQ9jP
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.s5pQsOIPqI
+ rm /tmp/tmp.J0qV3JQ9jP /tmp/tmp.s5pQsOIPqI
+ return 0
+ sleep 55
+ desc 'disable backups schedule'
+ set +o xtrace


-----------------------------------------------------------------------------------
disable backups schedule
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFE7FYHqmW
++ mktemp
+ local LAST_ERR=/tmp/tmp.gQXcYEMr1r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yFE7FYHqmW
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.gQXcYEMr1r
+ rm /tmp/tmp.yFE7FYHqmW /tmp/tmp.gQXcYEMr1r
+ return 0
+ '[' -z '' ']'
++ head -1
++ grep aws-s3
++ awk '{print$1}'
++ kubectl_bin get psmdb-backup
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8DxoQqpx8k
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wOjYBNVPiG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8DxoQqpx8k
++ cat /tmp/tmp.wOjYBNVPiG
++ rm /tmp/tmp.8DxoQqpx8k /tmp/tmp.wOjYBNVPiG
++ return 0
+ backup_name_aws=cron-some-name-20250324033200-qfm6f
++ head -1
++ awk '{print$1}'
++ kubectl_bin get psmdb-backup
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0MYqxx3umM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6Li6E3iMPk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ grep gcp-cs
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0MYqxx3umM
++ cat /tmp/tmp.6Li6E3iMPk
++ rm /tmp/tmp.0MYqxx3umM /tmp/tmp.6Li6E3iMPk
++ return 0
+ backup_name_gcp=cron-some-name-20250324033200-r7b57
++ head -1
++ kubectl_bin get psmdb-backup
+++ mktemp
++ awk '{print$1}'
++ grep azure-blob
++ local LAST_OUT=/tmp/tmp.DFO7ujPNFz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Yma2aCHoF3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DFO7ujPNFz
++ cat /tmp/tmp.Yma2aCHoF3
++ rm /tmp/tmp.DFO7ujPNFz /tmp/tmp.Yma2aCHoF3
++ return 0
+ backup_name_azure=cron-some-name-20250324033200-n7bkf
+ wait_backup cron-some-name-20250324033200-qfm6f
+ local backup_name=cron-some-name-20250324033200-qfm6f
+ local target_state=ready
+ set +o xtrace
waiting for cron-some-name-20250324033200-qfm6f to reach ready state.
+ wait_backup cron-some-name-20250324033200-r7b57
+ local backup_name=cron-some-name-20250324033200-r7b57
+ local target_state=ready
+ set +o xtrace
waiting for cron-some-name-20250324033200-r7b57 to reach ready state.
+ wait_backup cron-some-name-20250324033200-n7bkf
+ local backup_name=cron-some-name-20250324033200-n7bkf
+ local target_state=ready
+ set +o xtrace
waiting for cron-some-name-20250324033200-n7bkf to reach ready state.
++ head -1
++ kubectl_bin get psmdb-backup
++ awk '{print$1}'
++ grep minio
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NAyLtLlp0N
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UgLJTYsxDn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NAyLtLlp0N
++ cat /tmp/tmp.UgLJTYsxDn
++ rm /tmp/tmp.NAyLtLlp0N /tmp/tmp.UgLJTYsxDn
++ return 0
+ backup_name_minio=cron-some-name-20250324033200-g7k56
+ wait_backup cron-some-name-20250324033200-g7k56
+ local backup_name=cron-some-name-20250324033200-g7k56
+ local target_state=ready
+ set +o xtrace
waiting for cron-some-name-20250324033200-g7k56 to reach ready state.
+ sleep 5
+ echo -n 'checking backup count for every-min-minio...'
checking backup count for every-min-minio...+ check_backup_count every-min-minio 1
+ local ancestor=every-min-minio
+ local expected=1
++ grep ready
++ kubectl_bin get psmdb-backup -l percona.com/backup-ancestor=every-min-minio
+++ mktemp
++ wc -l
++ local LAST_OUT=/tmp/tmp.1trn3rbiEc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qBqVKcGFe4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup -l percona.com/backup-ancestor=every-min-minio
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1trn3rbiEc
++ cat /tmp/tmp.qBqVKcGFe4
++ rm /tmp/tmp.1trn3rbiEc /tmp/tmp.qBqVKcGFe4
++ return 0
+ local count=1
+ [[ 1 != 1 ]]
+ echo OK
OK
+ echo -n 'checking backup count for every-min-aws-s3...'
checking backup count for every-min-aws-s3...+ check_backup_count every-min-aws-s3 1
+ local ancestor=every-min-aws-s3
+ local expected=1
++ kubectl_bin get psmdb-backup -l percona.com/backup-ancestor=every-min-aws-s3
++ grep ready
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jQVAAox3t5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U6cocvg0hY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup -l percona.com/backup-ancestor=every-min-aws-s3
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jQVAAox3t5
++ cat /tmp/tmp.U6cocvg0hY
++ rm /tmp/tmp.jQVAAox3t5 /tmp/tmp.U6cocvg0hY
++ return 0
+ local count=1
+ [[ 1 != 1 ]]
+ echo OK
OK
+ echo -n 'checking backup count for every-min-gcp-cs...'
checking backup count for every-min-gcp-cs...+ check_backup_count every-min-gcp-cs 1
+ local ancestor=every-min-gcp-cs
+ local expected=1
++ kubectl_bin get psmdb-backup -l percona.com/backup-ancestor=every-min-gcp-cs
++ grep ready
++ wc -l
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3PpqWHOF31
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g1VfJ8KBy8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup -l percona.com/backup-ancestor=every-min-gcp-cs
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3PpqWHOF31
++ cat /tmp/tmp.g1VfJ8KBy8
++ rm /tmp/tmp.3PpqWHOF31 /tmp/tmp.g1VfJ8KBy8
++ return 0
+ local count=1
+ [[ 1 != 1 ]]
+ echo OK
OK
+ echo -n 'checking backup count for every-min-azure-blob...'
checking backup count for every-min-azure-blob...+ check_backup_count every-min-azure-blob 1
+ local ancestor=every-min-azure-blob
+ local expected=1
++ grep ready
++ wc -l
++ kubectl_bin get psmdb-backup -l percona.com/backup-ancestor=every-min-azure-blob
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aF7PvbeIYG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rbtA3UqFpa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup -l percona.com/backup-ancestor=every-min-azure-blob
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aF7PvbeIYG
++ cat /tmp/tmp.rbtA3UqFpa
++ rm /tmp/tmp.aF7PvbeIYG /tmp/tmp.rbtA3UqFpa
++ return 0
+ local count=1
+ [[ 1 != 1 ]]
+ echo OK
OK
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20250324033200-g7k56
++ local backup_name=cron-some-name-20250324033200-g7k56
++ kubectl_bin get psmdb-backup cron-some-name-20250324033200-g7k56 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|azure://||'
++ sed 's|s3://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YSa569NhQX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lswMe36DvP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20250324033200-g7k56 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YSa569NhQX
++ cat /tmp/tmp.lswMe36DvP
++ rm /tmp/tmp.YSa569NhQX /tmp/tmp.lswMe36DvP
++ return 0
+ backup_dest_minio=operator-testing/2025-03-24T03:32:36Z
+ grep myApp.test.gz
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-03-24T03:32:36Z/rs0/
++ mktemp
+ local LAST_OUT=/tmp/tmp.cedvjcpodT
++ mktemp
+ local LAST_ERR=/tmp/tmp.j3LX8EXQ6R
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-03-24T03:32:36Z/rs0/
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cedvjcpodT
+ cat /tmp/tmp.j3LX8EXQ6R
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: unable to upgrade connection: container aws-cli not found in pod aws-cli_scheduled-backup-12587
+ rm /tmp/tmp.cedvjcpodT /tmp/tmp.j3LX8EXQ6R
+ return 0
2025-03-24 03:32:39         55 myApp.test.gz
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IvOl2a8KFJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.H0LA90Rpg4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IvOl2a8KFJ
++ cat /tmp/tmp.H0LA90Rpg4
++ rm /tmp/tmp.IvOl2a8KFJ /tmp/tmp.H0LA90Rpg4
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZGUiMpPySt
++ mktemp
+ local LAST_ERR=/tmp/tmp.Bf479dei6n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZGUiMpPySt
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("7067c4e7-f09f-47c0-a219-7f351fb668af") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.Bf479dei6n
+ rm /tmp/tmp.ZGUiMpPySt /tmp/tmp.Bf479dei6n
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:16+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Aocouzd45
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RKUM8np2pz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2Aocouzd45
++ cat /tmp/tmp.RKUM8np2pz
++ rm /tmp/tmp.2Aocouzd45 /tmp/tmp.RKUM8np2pz
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.gxOqM6h6wq
++ mktemp
+ local LAST_ERR=/tmp/tmp.L5Qr14efg2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gxOqM6h6wq
+ cat /tmp/tmp.L5Qr14efg2
+ rm /tmp/tmp.gxOqM6h6wq /tmp/tmp.L5Qr14efg2
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:20+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LsoWpuLYwa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HF0MLPEHX0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LsoWpuLYwa
++ cat /tmp/tmp.HF0MLPEHX0
++ rm /tmp/tmp.LsoWpuLYwa /tmp/tmp.HF0MLPEHX0
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.J44mDHaZBG
++ mktemp
+ local LAST_ERR=/tmp/tmp.gWNg9u0E3B
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.J44mDHaZBG
+ cat /tmp/tmp.gWNg9u0E3B
+ rm /tmp/tmp.J44mDHaZBG /tmp/tmp.gWNg9u0E3B
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:25+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uR2fX38aRF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6YZwy0niJW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uR2fX38aRF
++ cat /tmp/tmp.6YZwy0niJW
++ rm /tmp/tmp.uR2fX38aRF /tmp/tmp.6YZwy0niJW
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.1rmN5LrErT
++ mktemp
+ local LAST_ERR=/tmp/tmp.ANg4EKbKXO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1rmN5LrErT
+ cat /tmp/tmp.ANg4EKbKXO
+ rm /tmp/tmp.1rmN5LrErT /tmp/tmp.ANg4EKbKXO
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ run_restore cron-some-name-20250324033200-g7k56
+ local backup_name=cron-some-name-20250324033200-g7k56
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20250324033200-g7k56/'
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20250324033200-g7k56/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.vZbErK3Ba8
++ mktemp
+ local LAST_ERR=/tmp/tmp.S4jqzzyst5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vZbErK3Ba8
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20250324033200-g7k56 created
+ cat /tmp/tmp.S4jqzzyst5
+ rm /tmp/tmp.vZbErK3Ba8 /tmp/tmp.S4jqzzyst5
+ return 0
+ wait_restore cron-some-name-20250324033200-g7k56 some-name
+ local backup_name=cron-some-name-20250324033200-g7k56
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-cron-some-name-20250324033200-g7k56 to reach ready state.......OK
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q4i5R23btJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ufvdo0reRO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q4i5R23btJ
++ cat /tmp/tmp.ufvdo0reRO
++ rm /tmp/tmp.Q4i5R23btJ /tmp/tmp.ufvdo0reRO
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:51+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BfC9lmNb7R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zGSL5vcG6n
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BfC9lmNb7R
++ cat /tmp/tmp.zGSL5vcG6n
++ rm /tmp/tmp.BfC9lmNb7R /tmp/tmp.zGSL5vcG6n
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.reaAh4NuTx
++ mktemp
+ local LAST_ERR=/tmp/tmp.VNpEEKqvj3
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.reaAh4NuTx
+ cat /tmp/tmp.VNpEEKqvj3
+ rm /tmp/tmp.reaAh4NuTx /tmp/tmp.VNpEEKqvj3
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:55+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.MxcNdv9pN0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SoY4D7WI2A
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MxcNdv9pN0
++ cat /tmp/tmp.SoY4D7WI2A
++ rm /tmp/tmp.MxcNdv9pN0 /tmp/tmp.SoY4D7WI2A
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SSGfNWsWOB
++ mktemp
+ local LAST_ERR=/tmp/tmp.QtRwPBZMPP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SSGfNWsWOB
+ cat /tmp/tmp.QtRwPBZMPP
+ rm /tmp/tmp.SSGfNWsWOB /tmp/tmp.QtRwPBZMPP
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:33:57+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ local LAST_OUT=/tmp/tmp.SPjPDz9okV
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NcKNtoKrOE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SPjPDz9okV
++ cat /tmp/tmp.NcKNtoKrOE
++ rm /tmp/tmp.SPjPDz9okV /tmp/tmp.NcKNtoKrOE
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.85J5x92uej
++ mktemp
+ local LAST_ERR=/tmp/tmp.3X9mPJFFUR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.85J5x92uej
+ cat /tmp/tmp.3X9mPJFFUR
+ rm /tmp/tmp.85J5x92uej /tmp/tmp.3X9mPJFFUR
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ '[' -z '' ']'
+ desc 'check backup and restore -- aws-s3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- aws-s3
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20250324033200-qfm6f
++ local backup_name=cron-some-name-20250324033200-qfm6f
++ kubectl_bin get psmdb-backup cron-some-name-20250324033200-qfm6f -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
+++ mktemp
++ sed -e 's/.json$//'
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.kBr3SBEgwg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p0hv7PWcOy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20250324033200-qfm6f -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kBr3SBEgwg
++ cat /tmp/tmp.p0hv7PWcOy
++ rm /tmp/tmp.kBr3SBEgwg /tmp/tmp.p0hv7PWcOy
++ return 0
+ backup_dest_aws=operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:25Z
+ gunzip
+ curl -s https://s3.amazonaws.com/operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:25Z/rs0/myApp.test.gz
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a44aRG9DOp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KEt8DOJ06M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a44aRG9DOp
++ cat /tmp/tmp.KEt8DOJ06M
++ rm /tmp/tmp.a44aRG9DOp /tmp/tmp.KEt8DOJ06M
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CJ2V3JQu2x
++ mktemp
+ local LAST_ERR=/tmp/tmp.gCQLgDi7va
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CJ2V3JQu2x
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("c0382246-c432-488f-ab2b-a7a0eba41869") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.gCQLgDi7va
+ rm /tmp/tmp.CJ2V3JQu2x /tmp/tmp.gCQLgDi7va
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:06+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Oi4olyD8Ow
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aSOKN8Zw73
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Oi4olyD8Ow
++ cat /tmp/tmp.aSOKN8Zw73
++ rm /tmp/tmp.Oi4olyD8Ow /tmp/tmp.aSOKN8Zw73
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BF5dTIY4Rv
++ mktemp
+ local LAST_ERR=/tmp/tmp.7DjOBQqep7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BF5dTIY4Rv
+ cat /tmp/tmp.7DjOBQqep7
+ rm /tmp/tmp.BF5dTIY4Rv /tmp/tmp.7DjOBQqep7
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:09+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.wbSA89GeNs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xt6s1i4T0h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wbSA89GeNs
++ cat /tmp/tmp.Xt6s1i4T0h
++ rm /tmp/tmp.wbSA89GeNs /tmp/tmp.Xt6s1i4T0h
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.v5JbYkABRH
++ mktemp
+ local LAST_ERR=/tmp/tmp.nS8otwdhLz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.v5JbYkABRH
+ cat /tmp/tmp.nS8otwdhLz
+ rm /tmp/tmp.v5JbYkABRH /tmp/tmp.nS8otwdhLz
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:12+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Exn8xpXSUp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MrMfUZP2H3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Exn8xpXSUp
++ cat /tmp/tmp.MrMfUZP2H3
++ rm /tmp/tmp.Exn8xpXSUp /tmp/tmp.MrMfUZP2H3
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HLN8gcFNlO
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZKOS0Odg68
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HLN8gcFNlO
+ cat /tmp/tmp.ZKOS0Odg68
+ rm /tmp/tmp.HLN8gcFNlO /tmp/tmp.ZKOS0Odg68
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ run_restore cron-some-name-20250324033200-qfm6f
+ local backup_name=cron-some-name-20250324033200-qfm6f
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6PZnJQes1h
++ mktemp
+ local LAST_ERR=/tmp/tmp.13ijyzT7WM
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20250324033200-qfm6f/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20250324033200-qfm6f/'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6PZnJQes1h
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20250324033200-qfm6f created
+ cat /tmp/tmp.13ijyzT7WM
+ rm /tmp/tmp.6PZnJQes1h /tmp/tmp.13ijyzT7WM
+ return 0
+ wait_restore cron-some-name-20250324033200-qfm6f some-name
+ local backup_name=cron-some-name-20250324033200-qfm6f
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-cron-some-name-20250324033200-qfm6f to reach ready state.......OK
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7p6WEckwcA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vScLA4tE1V
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7p6WEckwcA
++ cat /tmp/tmp.vScLA4tE1V
++ rm /tmp/tmp.7p6WEckwcA /tmp/tmp.vScLA4tE1V
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:39+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UQIhmvtt2H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mlaMP6QZHJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UQIhmvtt2H
++ cat /tmp/tmp.mlaMP6QZHJ
++ rm /tmp/tmp.UQIhmvtt2H /tmp/tmp.mlaMP6QZHJ
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ynQpQd4bs2
++ mktemp
+ local LAST_ERR=/tmp/tmp.UdYFtkADSP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ynQpQd4bs2
+ cat /tmp/tmp.UdYFtkADSP
+ rm /tmp/tmp.ynQpQd4bs2 /tmp/tmp.UdYFtkADSP
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:41+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.becu7bOR15
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dL0JD83q14
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.becu7bOR15
++ cat /tmp/tmp.dL0JD83q14
++ rm /tmp/tmp.becu7bOR15 /tmp/tmp.dL0JD83q14
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.3DrkiqbMDA
++ mktemp
+ local LAST_ERR=/tmp/tmp.wJuxXPa49o
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3DrkiqbMDA
+ cat /tmp/tmp.wJuxXPa49o
+ rm /tmp/tmp.3DrkiqbMDA /tmp/tmp.wJuxXPa49o
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:44+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BiI968eDKV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FrC9IhiLZp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BiI968eDKV
++ cat /tmp/tmp.FrC9IhiLZp
++ rm /tmp/tmp.BiI968eDKV /tmp/tmp.FrC9IhiLZp
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.AXyT3vUDtv
++ mktemp
+ local LAST_ERR=/tmp/tmp.5icbVBlytz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AXyT3vUDtv
+ cat /tmp/tmp.5icbVBlytz
+ rm /tmp/tmp.AXyT3vUDtv /tmp/tmp.5icbVBlytz
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ desc 'check backup and restore -- gcp-cs'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- gcp-cs
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20250324033200-r7b57
++ local backup_name=cron-some-name-20250324033200-r7b57
++ kubectl_bin get psmdb-backup cron-some-name-20250324033200-r7b57 -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YSRJzj7GzF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YFgqUmesXI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20250324033200-r7b57 -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.YSRJzj7GzF
++ cat /tmp/tmp.YFgqUmesXI
++ rm /tmp/tmp.YSRJzj7GzF /tmp/tmp.YFgqUmesXI
++ return 0
+ backup_dest_gcp=operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:00Z
+ curl -s https://storage.googleapis.com/operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:00Z/rs0/myApp.test.gz
+ gunzip
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VsTherhE4X
+++ mktemp
++ local LAST_ERR=/tmp/tmp.17cnbf5hks
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VsTherhE4X
++ cat /tmp/tmp.17cnbf5hks
++ rm /tmp/tmp.VsTherhE4X /tmp/tmp.17cnbf5hks
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.Cs6L8qN1AB
++ mktemp
+ local LAST_ERR=/tmp/tmp.jsa8Fa0hlo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Cs6L8qN1AB
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("58c7f749-3a7c-4eff-9748-a7828b910e51") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.jsa8Fa0hlo
+ rm /tmp/tmp.Cs6L8qN1AB /tmp/tmp.jsa8Fa0hlo
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:52+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RvKlPa1urd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fBZHUwAwiR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RvKlPa1urd
++ cat /tmp/tmp.fBZHUwAwiR
++ rm /tmp/tmp.RvKlPa1urd /tmp/tmp.fBZHUwAwiR
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.qgevXg4LUg
++ mktemp
+ local LAST_ERR=/tmp/tmp.9SX5XAoPo0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qgevXg4LUg
+ cat /tmp/tmp.9SX5XAoPo0
+ rm /tmp/tmp.qgevXg4LUg /tmp/tmp.9SX5XAoPo0
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:55+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4Bj2RUEnNr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n8bwfWURDK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4Bj2RUEnNr
++ cat /tmp/tmp.n8bwfWURDK
++ rm /tmp/tmp.4Bj2RUEnNr /tmp/tmp.n8bwfWURDK
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.thuxhJlYTv
++ mktemp
+ local LAST_ERR=/tmp/tmp.l0ibZRc11F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.thuxhJlYTv
+ cat /tmp/tmp.l0ibZRc11F
+ rm /tmp/tmp.thuxhJlYTv /tmp/tmp.l0ibZRc11F
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:34:59+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HLP7mM4iND
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SbezZo32CL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HLP7mM4iND
++ cat /tmp/tmp.SbezZo32CL
++ rm /tmp/tmp.HLP7mM4iND /tmp/tmp.SbezZo32CL
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UWovvyGIxF
++ mktemp
+ local LAST_ERR=/tmp/tmp.QbAodoy3Zw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UWovvyGIxF
+ cat /tmp/tmp.QbAodoy3Zw
+ rm /tmp/tmp.UWovvyGIxF /tmp/tmp.QbAodoy3Zw
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ run_restore cron-some-name-20250324033200-r7b57
+ local backup_name=cron-some-name-20250324033200-r7b57
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20250324033200-r7b57/'
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20250324033200-r7b57/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CrAcbfuIJC
++ mktemp
+ local LAST_ERR=/tmp/tmp.pDvCklhPHF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CrAcbfuIJC
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20250324033200-r7b57 created
+ cat /tmp/tmp.pDvCklhPHF
+ rm /tmp/tmp.CrAcbfuIJC /tmp/tmp.pDvCklhPHF
+ return 0
+ wait_restore cron-some-name-20250324033200-r7b57 some-name
+ local backup_name=cron-some-name-20250324033200-r7b57
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-cron-some-name-20250324033200-r7b57 to reach ready state.......OK
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uZj6OKIY9s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rJQuIVDiNc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uZj6OKIY9s
++ cat /tmp/tmp.rJQuIVDiNc
++ rm /tmp/tmp.uZj6OKIY9s /tmp/tmp.rJQuIVDiNc
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:26+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7h3diGXzQ3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LgFdM8QQE3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7h3diGXzQ3
++ cat /tmp/tmp.LgFdM8QQE3
++ rm /tmp/tmp.7h3diGXzQ3 /tmp/tmp.LgFdM8QQE3
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.SYzsVxdZ0h
++ mktemp
+ local LAST_ERR=/tmp/tmp.YNQvSIf9W4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SYzsVxdZ0h
+ cat /tmp/tmp.YNQvSIf9W4
+ rm /tmp/tmp.SYzsVxdZ0h /tmp/tmp.YNQvSIf9W4
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:29+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hmSSYn9u75
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EetibifOwe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hmSSYn9u75
++ cat /tmp/tmp.EetibifOwe
++ rm /tmp/tmp.hmSSYn9u75 /tmp/tmp.EetibifOwe
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.0CvBO2na7i
++ mktemp
+ local LAST_ERR=/tmp/tmp.vbZYvYRbCy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.0CvBO2na7i
+ cat /tmp/tmp.vbZYvYRbCy
+ rm /tmp/tmp.0CvBO2na7i /tmp/tmp.vbZYvYRbCy
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:33+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vx17YPivS6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jtLk0EyEpG
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vx17YPivS6
++ cat /tmp/tmp.jtLk0EyEpG
++ rm /tmp/tmp.vx17YPivS6 /tmp/tmp.jtLk0EyEpG
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.nEEtGckHbU
++ mktemp
+ local LAST_ERR=/tmp/tmp.tOQbFNn1rb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nEEtGckHbU
+ cat /tmp/tmp.tOQbFNn1rb
+ rm /tmp/tmp.nEEtGckHbU /tmp/tmp.tOQbFNn1rb
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ desc 'check backup and restore -- azure-blob'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- azure-blob
-----------------------------------------------------------------------------------

++ get_backup_dest cron-some-name-20250324033200-n7bkf
++ local backup_name=cron-some-name-20250324033200-n7bkf
++ kubectl_bin get psmdb-backup cron-some-name-20250324033200-n7bkf -o 'jsonpath={.status.destination}'
++ sed 's|azure://||'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bVxdi9O0Bk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2gztiiMi7Z
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup cron-some-name-20250324033200-n7bkf -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bVxdi9O0Bk
++ cat /tmp/tmp.2gztiiMi7Z
++ rm /tmp/tmp.bVxdi9O0Bk /tmp/tmp.2gztiiMi7Z
++ return 0
+ backup_dest_azure=operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:15Z
+ gunzip
+ curl -s https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-scheduled-backup/2025-03-24T03:32:15Z/rs0/myApp.test.gz
+ run_mongo 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-rs0.scheduled-backup-12587
+ local driver=mongodb+srv
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JmaH3HJUN6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QlmNroEDJe
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JmaH3HJUN6
++ cat /tmp/tmp.QlmNroEDJe
++ rm /tmp/tmp.JmaH3HJUN6 /tmp/tmp.QlmNroEDJe
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.oeKvEN93rQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.u4NkOxoftL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb+srv://myApp:myPass@some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.oeKvEN93rQ
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017,some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb&replicaSet=rs0&ssl=false
Implicit session: session { "id" : UUID("79f7f659-c7f9-4e15-9a03-55b55565fb17") }
Percona Server for MongoDB server version: v8.0.4-2
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.u4NkOxoftL
+ rm /tmp/tmp.oeKvEN93rQ /tmp/tmp.u4NkOxoftL
+ return 0
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:41+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XjJcHut5IC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PeVjsbu3Vh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XjJcHut5IC
++ cat /tmp/tmp.PeVjsbu3Vh
++ rm /tmp/tmp.XjJcHut5IC /tmp/tmp.PeVjsbu3Vh
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.mhTpojkj0o
++ mktemp
+ local LAST_ERR=/tmp/tmp.3TeOymBTHq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.mhTpojkj0o
+ cat /tmp/tmp.3TeOymBTHq
+ rm /tmp/tmp.mhTpojkj0o /tmp/tmp.3TeOymBTHq
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:44+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QAQzlL4RET
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8rgvN9PhSH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QAQzlL4RET
++ cat /tmp/tmp.8rgvN9PhSH
++ rm /tmp/tmp.QAQzlL4RET /tmp/tmp.8rgvN9PhSH
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.knSfoN3LkL
++ mktemp
+ local LAST_ERR=/tmp/tmp.yLrraRATGa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.knSfoN3LkL
+ cat /tmp/tmp.yLrraRATGa
+ rm /tmp/tmp.knSfoN3LkL /tmp/tmp.yLrraRATGa
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 -2nd
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=-2nd
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:35:48+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NRzgBgySZ2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mnpcjw6Mcc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NRzgBgySZ2
++ cat /tmp/tmp.mnpcjw6Mcc
++ rm /tmp/tmp.NRzgBgySZ2 /tmp/tmp.mnpcjw6Mcc
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.53drHbY6ye
++ mktemp
+ local LAST_ERR=/tmp/tmp.lW7cE5OjGN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.53drHbY6ye
+ cat /tmp/tmp.lW7cE5OjGN
+ rm /tmp/tmp.53drHbY6ye /tmp/tmp.lW7cE5OjGN
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find-2nd.json /tmp/tmp.w55tYLOlFW/find-2nd
+ run_restore cron-some-name-20250324033200-n7bkf
+ local backup_name=cron-some-name-20250324033200-n7bkf
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20250324033200-n7bkf/'
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20250324033200-n7bkf/'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/restore.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.FstKXgq3NC
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ji9CokzgLe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FstKXgq3NC
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20250324033200-n7bkf created
+ cat /tmp/tmp.Ji9CokzgLe
+ rm /tmp/tmp.FstKXgq3NC /tmp/tmp.Ji9CokzgLe
+ return 0
+ wait_restore cron-some-name-20250324033200-n7bkf some-name
+ local backup_name=cron-some-name-20250324033200-n7bkf
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-cron-some-name-20250324033200-n7bkf to reach ready state.......OK
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Fnn6BGlGMM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6SYCcuelh0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Fnn6BGlGMM
++ cat /tmp/tmp.6SYCcuelh0
++ rm /tmp/tmp.Fnn6BGlGMM /tmp/tmp.6SYCcuelh0
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:36:12+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ILFgDBJ16I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qGYMityAPi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ILFgDBJ16I
++ cat /tmp/tmp.qGYMityAPi
++ rm /tmp/tmp.ILFgDBJ16I /tmp/tmp.qGYMityAPi
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.p0FXamrnra
++ mktemp
+ local LAST_ERR=/tmp/tmp.XCRJh25Uty
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.p0FXamrnra
+ cat /tmp/tmp.XCRJh25Uty
+ rm /tmp/tmp.p0FXamrnra /tmp/tmp.XCRJh25Uty
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:36:14+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mbPSoYpvNN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hCPIdEwNJm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mbPSoYpvNN
++ cat /tmp/tmp.hCPIdEwNJm
++ rm /tmp/tmp.mbPSoYpvNN /tmp/tmp.hCPIdEwNJm
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9JRwdBQ7IA
++ mktemp
+ local LAST_ERR=/tmp/tmp.1DJ4ZbpIs6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9JRwdBQ7IA
+ cat /tmp/tmp.1DJ4ZbpIs6
+ rm /tmp/tmp.9JRwdBQ7IA /tmp/tmp.1DJ4ZbpIs6
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:36:16+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ox4CcyiKb0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5IZOCLOvP3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ox4CcyiKb0
++ cat /tmp/tmp.5IZOCLOvP3
++ rm /tmp/tmp.Ox4CcyiKb0 /tmp/tmp.5IZOCLOvP3
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.C3yNIa05ME
++ mktemp
+ local LAST_ERR=/tmp/tmp.I3eIiEBBfm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C3yNIa05ME
+ cat /tmp/tmp.I3eIiEBBfm
+ rm /tmp/tmp.C3yNIa05ME /tmp/tmp.I3eIiEBBfm
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ desc 'add physical backup schedule, wait for the first backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
add physical backup schedule, wait for the first backup
-----------------------------------------------------------------------------------

+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"'
+ yq eval '.spec.upgradeOptions.apply="Never"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ local LAST_OUT=/tmp/tmp.1TTiZG0W0S
++ mktemp
+ local LAST_ERR=/tmp/tmp.R0J6yl6VB6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1TTiZG0W0S
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.R0J6yl6VB6
+ rm /tmp/tmp.1TTiZG0W0S /tmp/tmp.R0J6yl6VB6
+ return 0
+ sleep 55
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/some-name-rs0.yml
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1872-66b64516"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.nihLqX9ZzU
++ mktemp
+ local LAST_ERR=/tmp/tmp.jj5dIAp7cz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nihLqX9ZzU
perconaservermongodb.psmdb.percona.com/some-name configured
+ cat /tmp/tmp.jj5dIAp7cz
+ rm /tmp/tmp.nihLqX9ZzU /tmp/tmp.jj5dIAp7cz
+ return 0
++ kubectl_bin get psmdb-backup
++ grep minio
++ tail -1
++ awk '{print $1}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K86hVyYpm5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HGonUUe2DO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.K86hVyYpm5
++ cat /tmp/tmp.HGonUUe2DO
++ rm /tmp/tmp.K86hVyYpm5 /tmp/tmp.HGonUUe2DO
++ return 0
+ backup_name_minio=cron-some-name-20250324033700-7n5hs
+ wait_backup cron-some-name-20250324033700-7n5hs
+ local backup_name=cron-some-name-20250324033700-7n5hs
+ local target_state=ready
+ set +o xtrace
waiting for cron-some-name-20250324033700-7n5hs to reach ready state.
+ sleep 5
+ desc 'check backup and restore -- minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
check backup and restore -- minio
-----------------------------------------------------------------------------------

+ run_restore cron-some-name-20250324033700-7n5hs 3
+ local backup_name=cron-some-name-20250324033700-7n5hs
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/conf/restore.yml
+ /usr/bin/sed -e 's/backupName:/backupName: cron-some-name-20250324033700-7n5hs/'
+ /usr/bin/sed -e 's/name:/name: restore-cron-some-name-20250324033700-7n5hs/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hcVTZjACmg
++ mktemp
+ local LAST_ERR=/tmp/tmp.ts6aSy2Fw2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hcVTZjACmg
perconaservermongodbrestore.psmdb.percona.com/restore-cron-some-name-20250324033700-7n5hs created
+ cat /tmp/tmp.ts6aSy2Fw2
+ rm /tmp/tmp.hcVTZjACmg /tmp/tmp.ts6aSy2Fw2
+ return 0
+ wait_restore cron-some-name-20250324033700-7n5hs some-name
+ local backup_name=cron-some-name-20250324033700-7n5hs
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=1
+ local wait_time=1780
+ local ok_if_ready=0
+ set +o xtrace
waiting psmdb-restore/restore-cron-some-name-20250324033700-7n5hs to reach ready state.............................................................................................................................OK
+ '[' 1 -eq 1 ']'
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XUzR6ucmx0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iP77S9qH6g
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.XUzR6ucmx0
++ cat /tmp/tmp.iP77S9qH6g
++ rm /tmp/tmp.XUzR6ucmx0 /tmp/tmp.iP77S9qH6g
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eJ7RJU9vXz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oPl6Yu5ard
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eJ7RJU9vXz
++ cat /tmp/tmp.oPl6Yu5ard
++ rm /tmp/tmp.eJ7RJU9vXz /tmp/tmp.oPl6Yu5ard
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xvBB44Yu93
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IM1d7HhWfP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xvBB44Yu93
++ cat /tmp/tmp.IM1d7HhWfP
++ rm /tmp/tmp.xvBB44Yu93 /tmp/tmp.IM1d7HhWfP
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1v9urJ2K4C
+++ mktemp
++ local LAST_ERR=/tmp/tmp.A4GHG4Dyjx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1v9urJ2K4C
++ cat /tmp/tmp.A4GHG4Dyjx
++ rm /tmp/tmp.1v9urJ2K4C /tmp/tmp.A4GHG4Dyjx
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TWO5qoLDwe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wg3mwVV7lP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.TWO5qoLDwe
++ cat /tmp/tmp.wg3mwVV7lP
++ rm /tmp/tmp.TWO5qoLDwe /tmp/tmp.wg3mwVV7lP
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zqNHigqko5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wsgyr92R0C
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zqNHigqko5
++ cat /tmp/tmp.wsgyr92R0C
++ rm /tmp/tmp.zqNHigqko5 /tmp/tmp.wsgyr92R0C
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5jvlIPTcow
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BqT8UAZxeR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5jvlIPTcow
++ cat /tmp/tmp.BqT8UAZxeR
++ rm /tmp/tmp.5jvlIPTcow /tmp/tmp.BqT8UAZxeR
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MK4FnRC500
+++ mktemp
++ local LAST_ERR=/tmp/tmp.E8xOYN1eeH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MK4FnRC500
++ cat /tmp/tmp.E8xOYN1eeH
++ rm /tmp/tmp.MK4FnRC500 /tmp/tmp.E8xOYN1eeH
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SK7QIUv5LB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.otLh1zEmV4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SK7QIUv5LB
++ cat /tmp/tmp.otLh1zEmV4
++ rm /tmp/tmp.SK7QIUv5LB /tmp/tmp.otLh1zEmV4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wQNAqZZ9qb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.P40uILiuhu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wQNAqZZ9qb
++ cat /tmp/tmp.P40uILiuhu
++ rm /tmp/tmp.wQNAqZZ9qb /tmp/tmp.P40uILiuhu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CvHsB3uGfq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9jdy0uS8jS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CvHsB3uGfq
++ cat /tmp/tmp.9jdy0uS8jS
++ rm /tmp/tmp.CvHsB3uGfq /tmp/tmp.9jdy0uS8jS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Cx9GMwGpJ9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iyMnUdJZQQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Cx9GMwGpJ9
++ cat /tmp/tmp.iyMnUdJZQQ
++ rm /tmp/tmp.Cx9GMwGpJ9 /tmp/tmp.iyMnUdJZQQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MPPQz7a0Ud
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1NFEJOOaNc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.MPPQz7a0Ud
++ cat /tmp/tmp.1NFEJOOaNc
++ rm /tmp/tmp.MPPQz7a0Ud /tmp/tmp.1NFEJOOaNc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 32 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qkFQO5ZXuF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IvV1TFV2YP
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qkFQO5ZXuF
++ cat /tmp/tmp.IvV1TFV2YP
++ rm /tmp/tmp.qkFQO5ZXuF /tmp/tmp.IvV1TFV2YP
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ compare_mongo_cmd find myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:43:25+0000] running db.test.find() in myApp
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 mongodb ''
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nFvGO3wUdU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pXhcyNo3Cm
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nFvGO3wUdU
++ cat /tmp/tmp.pXhcyNo3Cm
++ rm /tmp/tmp.nFvGO3wUdU /tmp/tmp.pXhcyNo3Cm
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.CdaxvT4F3A
++ mktemp
+ local LAST_ERR=/tmp/tmp.eTgciNDHDp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-0.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CdaxvT4F3A
+ cat /tmp/tmp.eTgciNDHDp
+ rm /tmp/tmp.CdaxvT4F3A /tmp/tmp.eTgciNDHDp
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:43:28+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.BDyvoo0Lnd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zfTzGYAKeO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BDyvoo0Lnd
++ cat /tmp/tmp.zfTzGYAKeO
++ rm /tmp/tmp.BDyvoo0Lnd /tmp/tmp.zfTzGYAKeO
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.WqVrvBPTxw
++ mktemp
+ local LAST_ERR=/tmp/tmp.km0D8XqSPR
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-1.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WqVrvBPTxw
+ cat /tmp/tmp.km0D8XqSPR
+ rm /tmp/tmp.WqVrvBPTxw /tmp/tmp.km0D8XqSPR
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ compare_mongo_cmd find myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local command=find
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local postfix=
+ local suffix=
+ local database=myApp
+ local collection=test
+ local sort=
+ local 'full_command=db.test.find()'
+ [[ ! -z '' ]]
+ log 'running db.test.find() in myApp'
+ set +o xtrace
[2025-03-24T03:43:31+0000] running db.test.find() in myApp
+ egrep -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongo 'use myApp\n db.test.find()' myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ local LAST_OUT=/tmp/tmp.KcTU1rHFxZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6KcPHzUjZ5
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KcTU1rHFxZ
++ cat /tmp/tmp.6KcPHzUjZ5
++ rm /tmp/tmp.KcTU1rHFxZ /tmp/tmp.6KcPHzUjZ5
++ return 0
+ local client_container=psmdb-client-874f474b6-c44mf
+ local mongo_flag=
+ [[ myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587 == *cfg* ]]
+ replica_set=rs0
+ kubectl_bin exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.YyfeNC9pw6
++ mktemp
+ local LAST_ERR=/tmp/tmp.LBcIe8tvxB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-874f474b6-c44mf -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-rs0-2.some-name-rs0.scheduled-backup-12587.svc.cluster.local/admin?ssl=false\&replicaSet=rs0 '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YyfeNC9pw6
+ cat /tmp/tmp.LBcIe8tvxB
+ rm /tmp/tmp.YyfeNC9pw6 /tmp/tmp.LBcIe8tvxB
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/e2e-tests/scheduled-backup/compare/find.json /tmp/tmp.w55tYLOlFW/find
+ sleep 60
+ unlabel_node
+ desc 'remove labels'
+ set +o xtrace


-----------------------------------------------------------------------------------
remove labels
-----------------------------------------------------------------------------------

+ kubectl_bin label nodes gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq backupWorker- --overwrite
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZpXjyCj4UM
++ mktemp
+ local LAST_ERR=/tmp/tmp.tbC8j5YYdQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label nodes gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq backupWorker- --overwrite
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZpXjyCj4UM
node/gke-jen-psmdb-1872-66b64-default-pool-68ec5561-0xlq labeled
+ cat /tmp/tmp.tbC8j5YYdQ
+ rm /tmp/tmp.ZpXjyCj4UM /tmp/tmp.tbC8j5YYdQ
+ return 0
+ destroy scheduled-backup-12587
+ local namespace=scheduled-backup-12587
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.xn6BAhYIHk
++ mktemp
+ local LAST_ERR=/tmp/tmp.0otDp0Quns
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xn6BAhYIHk
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.0otDp0Quns
+ rm /tmp/tmp.xn6BAhYIHk /tmp/tmp.0otDp0Quns
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n scheduled-backup-12587 cron-some-name-20250324033200-n7bkf --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "cron-some-name-20250324033200-n7bkf" not found
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.aWZwfPFjCx
++ mktemp
+ local LAST_ERR=/tmp/tmp.bIEmBOk1Xz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.aWZwfPFjCx
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.bIEmBOk1Xz
+ rm /tmp/tmp.aWZwfPFjCx /tmp/tmp.bIEmBOk1Xz
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.JkLdYjfdP0
++ mktemp
+ local LAST_ERR=/tmp/tmp.jgRleHoxBj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JkLdYjfdP0
+ cat /tmp/tmp.jgRleHoxBj
+ rm /tmp/tmp.JkLdYjfdP0 /tmp/tmp.jgRleHoxBj
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.gYuiE2VhZl
++ mktemp
+ local LAST_ERR=/tmp/tmp.oW4SQKJmam
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gYuiE2VhZl
+ cat /tmp/tmp.oW4SQKJmam
+ rm /tmp/tmp.gYuiE2VhZl /tmp/tmp.oW4SQKJmam
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.8NOn36m7j9
++ mktemp
+ local LAST_ERR=/tmp/tmp.DZXNuJySzz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1872/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8NOn36m7j9
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.DZXNuJySzz
+ rm /tmp/tmp.8NOn36m7j9 /tmp/tmp.DZXNuJySzz
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.CssQ9nh4gs
++ mktemp
+ local LAST_ERR=/tmp/tmp.641sZajTWX
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.CssQ9nh4gs
+ cat /tmp/tmp.641sZajTWX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.CssQ9nh4gs
+ cat /tmp/tmp.641sZajTWX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.CssQ9nh4gs
+ cat /tmp/tmp.641sZajTWX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.CssQ9nh4gs
+ cat /tmp/tmp.641sZajTWX
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.3/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.CssQ9nh4gs /tmp/tmp.641sZajTWX
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.w55tYLOlFW
+ kubectl_bin delete --grace-period=0 --force=true namespace scheduled-backup-12587
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.ukmcWAVOE6
++ mktemp
+ local LAST_ERR=/tmp/tmp.6AqTV0jSAW
+ local exit_status=0
+ local timeout=4
+ local LAST_OUT=/tmp/tmp.iZFC8Xb55b
++ mktemp
+ local LAST_ERR=/tmp/tmp.2zR8NLgcGI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace scheduled-backup-12587
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator