Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/logs/demand-backup-incremental-sharded.log
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-incremental-sharded-608
+ local ns=demand-backup-incremental-sharded-608
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.Uy1OeIezZ1
++ mktemp
+ local LAST_ERR=/tmp/tmp.6EiREbE1Z2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Uy1OeIezZ1
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.6EiREbE1Z2
+ rm /tmp/tmp.Uy1OeIezZ1 /tmp/tmp.6EiREbE1Z2
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-5216 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-5216 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-5216 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-5216 backup-minio-not-base-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-incremental-sharded-5216 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.b7l3buJ8Zl
++ mktemp
+ local LAST_ERR=/tmp/tmp.hirSlr6VHz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.b7l3buJ8Zl
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.hirSlr6VHz
+ rm /tmp/tmp.b7l3buJ8Zl /tmp/tmp.hirSlr6VHz
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.UY4cT9DKWN
++ mktemp
+ local LAST_ERR=/tmp/tmp.X5uv1bpeZQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UY4cT9DKWN
+ cat /tmp/tmp.X5uv1bpeZQ
+ rm /tmp/tmp.UY4cT9DKWN /tmp/tmp.X5uv1bpeZQ
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.jElT4ALMBJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.yAs56NpI7z
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jElT4ALMBJ
+ cat /tmp/tmp.yAs56NpI7z
+ rm /tmp/tmp.jElT4ALMBJ /tmp/tmp.yAs56NpI7z
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.ne2S6B5jGk
++ mktemp
+ local LAST_ERR=/tmp/tmp.DA0ZdHJoyr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ne2S6B5jGk
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.DA0ZdHJoyr
+ rm /tmp/tmp.ne2S6B5jGk /tmp/tmp.DA0ZdHJoyr
+ return 0
+ check_crd_for_deletion PR-1926-d4d89df0
+ local git_tag=PR-1926-d4d89df0
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1926-d4d89df0/deploy/crd.yaml
++ /usr/bin/sed s/---//g
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XzQKVuMMGS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tCY8TLupJ0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XzQKVuMMGS
++ cat /tmp/tmp.tCY8TLupJ0
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XzQKVuMMGS
++ cat /tmp/tmp.tCY8TLupJ0
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.XzQKVuMMGS
++ cat /tmp/tmp.tCY8TLupJ0
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.XzQKVuMMGS
++ cat /tmp/tmp.tCY8TLupJ0
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.XzQKVuMMGS /tmp/tmp.tCY8TLupJ0
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ xargs kubectl delete ns
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.V3DhKARQ9N
++ mktemp
+ local LAST_OUT=/tmp/tmp.tQu3oHIi4q
++ mktemp
+ local LAST_ERR=/tmp/tmp.lgHEfCwn2Q
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.rMyH5ii15F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tQu3oHIi4q
+ cat /tmp/tmp.rMyH5ii15F
+ rm /tmp/tmp.tQu3oHIi4q /tmp/tmp.rMyH5ii15F
+ return 0
namespace "demand-backup-incremental-sharded-5216" deleted
namespace "gke-managed-cim" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V3DhKARQ9N
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.lgHEfCwn2Q
+ rm /tmp/tmp.V3DhKARQ9N /tmp/tmp.lgHEfCwn2Q
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ANgkSg48eJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.oDtfrbHAT1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ANgkSg48eJ
+ cat /tmp/tmp.oDtfrbHAT1
+ rm /tmp/tmp.ANgkSg48eJ /tmp/tmp.oDtfrbHAT1
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YlHdjE6WL5
++ mktemp
+ local LAST_ERR=/tmp/tmp.w9YibQFaxs
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YlHdjE6WL5
namespace/psmdb-operator created
+ cat /tmp/tmp.w9YibQFaxs
+ rm /tmp/tmp.YlHdjE6WL5 /tmp/tmp.w9YibQFaxs
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gPvczWKAub
+++ mktemp
++ local LAST_ERR=/tmp/tmp.c6s1zcnRqo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gPvczWKAub
++ cat /tmp/tmp.c6s1zcnRqo
++ rm /tmp/tmp.gPvczWKAub /tmp/tmp.c6s1zcnRqo
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.P2zf5SElnM
++ mktemp
+ local LAST_ERR=/tmp/tmp.0EVXXUgvok
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.P2zf5SElnM
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4" modified.
+ cat /tmp/tmp.0EVXXUgvok
+ rm /tmp/tmp.P2zf5SElnM /tmp/tmp.0EVXXUgvok
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Odc2OuhqzL
++ mktemp
+ local LAST_ERR=/tmp/tmp.OU97h0PUCe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Odc2OuhqzL
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.OU97h0PUCe
+ rm /tmp/tmp.Odc2OuhqzL /tmp/tmp.OU97h0PUCe
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.PJP0obiqjb
++ mktemp
+ local LAST_ERR=/tmp/tmp.5yjHHmXIL8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PJP0obiqjb
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.5yjHHmXIL8
+ rm /tmp/tmp.PJP0obiqjb /tmp/tmp.5yjHHmXIL8
+ return 0
+ kubectl_bin apply -f -
+ yq eval '
			(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1926-d4d89df0") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.13vV583wXA
++ mktemp
+ local LAST_ERR=/tmp/tmp.kob80kI4yB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.13vV583wXA
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.kob80kI4yB
+ rm /tmp/tmp.13vV583wXA /tmp/tmp.kob80kI4yB
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rikeWcsVsO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cRwB8WkdKR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rikeWcsVsO
++ cat /tmp/tmp.cRwB8WkdKR
++ rm /tmp/tmp.rikeWcsVsO /tmp/tmp.cRwB8WkdKR
++ return 0
+ wait_pod percona-server-mongodb-operator-c5895859f-fqgmw
+ local pod=percona-server-mongodb-operator-c5895859f-fqgmw
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-c5895859f-fqgmw to be ready...OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4yHSWv5DhC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Qvb3MBQtR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4yHSWv5DhC
++ cat /tmp/tmp.9Qvb3MBQtR
++ rm /tmp/tmp.4yHSWv5DhC /tmp/tmp.9Qvb3MBQtR
++ return 0
+ kubectl_bin logs percona-server-mongodb-operator-c5895859f-fqgmw
++ mktemp
+ local LAST_OUT=/tmp/tmp.5M1zeOlssU
++ mktemp
+ local LAST_ERR=/tmp/tmp.qFzPbuhKRU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs percona-server-mongodb-operator-c5895859f-fqgmw
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5M1zeOlssU
+ cat /tmp/tmp.qFzPbuhKRU
+ rm /tmp/tmp.5M1zeOlssU /tmp/tmp.qFzPbuhKRU
+ return 0
2025-06-11T12:07:08.469Z	INFO	setup	Manager starting up	{"gitCommit": "d4d89df06387c83c75ec97fa349bfd60d81a9afc", "gitBranch": "PR-1926-d4d89df0", "buildTime": "", "goVersion": "go1.24.4", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-incremental-sharded-608
+ local namespace=demand-backup-incremental-sharded-608
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-incremental-sharded-608'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-incremental-sharded-608
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-incremental-sharded-608 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.C672XqSWyX
+ local LAST_OUT=/tmp/tmp.SLWYVL9upJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8Hr4jM8erW
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.gFuv0MiFEs
+ local exit_status=0
+ local timeout=4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-incremental-sharded-608 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.C672XqSWyX
+ cat /tmp/tmp.8Hr4jM8erW
+ rm /tmp/tmp.C672XqSWyX /tmp/tmp.8Hr4jM8erW
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SLWYVL9upJ
+ cat /tmp/tmp.gFuv0MiFEs
+ rm /tmp/tmp.SLWYVL9upJ /tmp/tmp.gFuv0MiFEs
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-incremental-sharded-608
++ mktemp
+ local LAST_OUT=/tmp/tmp.GKDfq5jVMc
++ mktemp
+ local LAST_ERR=/tmp/tmp.QTt01JgLZf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-incremental-sharded-608
namespace "gke-managed-cim" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GKDfq5jVMc
+ cat /tmp/tmp.QTt01JgLZf
+ rm /tmp/tmp.GKDfq5jVMc /tmp/tmp.QTt01JgLZf
+ return 0
+ desc 'create namespace demand-backup-incremental-sharded-608'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-incremental-sharded-608
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-incremental-sharded-608
++ mktemp
+ local LAST_OUT=/tmp/tmp.JuzwmnttUT
++ mktemp
+ local LAST_ERR=/tmp/tmp.49Do05dP8s
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-incremental-sharded-608
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JuzwmnttUT
namespace/demand-backup-incremental-sharded-608 created
+ cat /tmp/tmp.49Do05dP8s
+ rm /tmp/tmp.JuzwmnttUT /tmp/tmp.49Do05dP8s
+ return 0
+ set_kube_ctx demand-backup-incremental-sharded-608
+ local namespace=demand-backup-incremental-sharded-608
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4EQUJlilPB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dtXWKxKwtI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
namespace "gmp-system" deleted
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4EQUJlilPB
++ cat /tmp/tmp.dtXWKxKwtI
++ rm /tmp/tmp.4EQUJlilPB /tmp/tmp.dtXWKxKwtI
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4 --namespace=demand-backup-incremental-sharded-608
++ mktemp
+ local LAST_OUT=/tmp/tmp.pCkyHZDn8A
++ mktemp
+ local LAST_ERR=/tmp/tmp.xaChoZmUDb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4 --namespace=demand-backup-incremental-sharded-608
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pCkyHZDn8A
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1926-d4d89df0-31-cluster4" modified.
+ cat /tmp/tmp.xaChoZmUDb
+ rm /tmp/tmp.pCkyHZDn8A /tmp/tmp.xaChoZmUDb
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Jun 11 12:07:31 2025
NAMESPACE: demand-backup-incremental-sharded-608
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-incremental-sharded-608.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-incremental-sharded-608 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-incremental-sharded-608

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-incremental-sharded-608 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-incremental-sharded-608 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1FketuWDMN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.s0QR2AOtIH
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1FketuWDMN
++ cat /tmp/tmp.s0QR2AOtIH
++ rm /tmp/tmp.1FketuWDMN /tmp/tmp.s0QR2AOtIH
++ return 0
+ MINIO_POD=minio-service-86dfccd949-5r6vd
+ wait_pod minio-service-86dfccd949-5r6vd
+ local pod=minio-service-86dfccd949-5r6vd
+ set +o xtrace
waiting for pod/minio-service-86dfccd949-5r6vd to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-608.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.vuxVCtu3MK
++ mktemp
+ local LAST_ERR=/tmp/tmp.5xpe0FmNwC
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-incremental-sharded-608.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vuxVCtu3MK
service/minio-service created
+ cat /tmp/tmp.5xpe0FmNwC
+ rm /tmp/tmp.vuxVCtu3MK /tmp/tmp.5xpe0FmNwC
+ return 0
+ create_minio_bucket operator-testing
+ local bucket=operator-testing
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.TUIDHVY9Jz
++ mktemp
+ local LAST_ERR=/tmp/tmp.FiUXsHxMbh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key 		AWS_SECRET_ACCESS_KEY=some-secret-key 		AWS_DEFAULT_REGION=us-east-1         /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TUIDHVY9Jz
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.FiUXsHxMbh
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-608
+ rm /tmp/tmp.TUIDHVY9Jz /tmp/tmp.FiUXsHxMbh
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PIwJnrwbho
++ mktemp
+ local LAST_ERR=/tmp/tmp.3WmzQ52WSI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PIwJnrwbho
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.3WmzQ52WSI
+ rm /tmp/tmp.PIwJnrwbho /tmp/tmp.3WmzQ52WSI
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------

+ log 'Creating PSMDB cluster'
+ set +o xtrace
[2025-06-11T12:08:23+0000] Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xCNeWM24TL
++ mktemp
+ local LAST_ERR=/tmp/tmp.EEwKphPJwN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xCNeWM24TL
secret/some-users created
+ cat /tmp/tmp.EEwKphPJwN
+ rm /tmp/tmp.xCNeWM24TL /tmp/tmp.EEwKphPJwN
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/some-name-sharded.yml
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ local LAST_OUT=/tmp/tmp.zmOdsm7U30
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1926-d4d89df0"'
++ mktemp
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
+ local LAST_ERR=/tmp/tmp.Elbcom6gpE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.zmOdsm7U30
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.Elbcom6gpE
+ rm /tmp/tmp.zmOdsm7U30 /tmp/tmp.Elbcom6gpE
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nLKI2CHog6
++ mktemp
+ local LAST_ERR=/tmp/tmp.ma6NwvJhCz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nLKI2CHog6
deployment.apps/psmdb-client created
+ cat /tmp/tmp.ma6NwvJhCz
+ rm /tmp/tmp.nLKI2CHog6 /tmp/tmp.ma6NwvJhCz
+ return 0
+ log 'check if all pods started'
+ set +o xtrace
[2025-06-11T12:08:28+0000] check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready.......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready......OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8vVvfgmny3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MvomHVJe8J
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8vVvfgmny3
++ cat /tmp/tmp.MvomHVJe8J
++ rm /tmp/tmp.8vVvfgmny3 /tmp/tmp.MvomHVJe8J
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready.....OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bwXEsxeTI9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1Z9w99MIcc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bwXEsxeTI9
++ cat /tmp/tmp.1Z9w99MIcc
++ rm /tmp/tmp.bwXEsxeTI9 /tmp/tmp.1Z9w99MIcc
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mnSv0kNQvS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qJFkYhtAXQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mnSv0kNQvS
++ cat /tmp/tmp.qJFkYhtAXQ
++ rm /tmp/tmp.mnSv0kNQvS /tmp/tmp.qJFkYhtAXQ
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..........................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0gE8sal8IB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VFIumd6yuh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0gE8sal8IB
++ cat /tmp/tmp.VFIumd6yuh
++ rm /tmp/tmp.0gE8sal8IB /tmp/tmp.VFIumd6yuh
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IaOCewtijm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n5dCrbAeQf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.IaOCewtijm
++ cat /tmp/tmp.n5dCrbAeQf
++ rm /tmp/tmp.IaOCewtijm /tmp/tmp.n5dCrbAeQf
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.34evabM8t2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d6Ve0Zsr67
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.34evabM8t2
++ cat /tmp/tmp.d6Ve0Zsr67
++ rm /tmp/tmp.34evabM8t2 /tmp/tmp.d6Ve0Zsr67
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Z8jtpD77IO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZGBfeTaB3M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Z8jtpD77IO
++ cat /tmp/tmp.ZGBfeTaB3M
++ rm /tmp/tmp.Z8jtpD77IO /tmp/tmp.ZGBfeTaB3M
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bQmwIBxVtb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cC284ZveKy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.bQmwIBxVtb
++ cat /tmp/tmp.cC284ZveKy
++ rm /tmp/tmp.bQmwIBxVtb /tmp/tmp.cC284ZveKy
++ return 0
+ [[ '' == \t\r\u\e ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iJZsiPm3cr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lfSnDxH2vE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].hidden.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iJZsiPm3cr
++ cat /tmp/tmp.lfSnDxH2vE
++ rm /tmp/tmp.iJZsiPm3cr /tmp/tmp.lfSnDxH2vE
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PcNlvLwu3K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XmrDY4RpmS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PcNlvLwu3K
++ cat /tmp/tmp.XmrDY4RpmS
++ rm /tmp/tmp.PcNlvLwu3K /tmp/tmp.XmrDY4RpmS
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qBlaFvBfTD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0OaIEXphsM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qBlaFvBfTD
++ cat /tmp/tmp.0OaIEXphsM
++ rm /tmp/tmp.qBlaFvBfTD /tmp/tmp.0OaIEXphsM
++ return 0
+ lbEndpoint=34.30.209.98
+ '[' -z 34.30.209.98 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RUYG5WECg4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WxgM4L5L7M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RUYG5WECg4
++ cat /tmp/tmp.WxgM4L5L7M
++ rm /tmp/tmp.RUYG5WECg4 /tmp/tmp.WxgM4L5L7M
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.EEKIcZVaV8
++ mktemp
+ local LAST_ERR=/tmp/tmp.mVJYIC11QG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.EEKIcZVaV8
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("cea51541-331f-4458-a81d-6da9be747017") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
Successfully added user: {
	"user" : "myApp",
	"roles" : [
		{
			"db" : "myApp",
			"role" : "readWrite"
		}
	]
}
bye
+ cat /tmp/tmp.mVJYIC11QG
+ rm /tmp/tmp.EEKIcZVaV8 /tmp/tmp.mVJYIC11QG
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SNtu9Fpki7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pU6Gk9F6Wl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SNtu9Fpki7
++ cat /tmp/tmp.pU6Gk9F6Wl
++ rm /tmp/tmp.SNtu9Fpki7 /tmp/tmp.pU6Gk9F6Wl
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.n8Fv35FBto
++ mktemp
+ local LAST_ERR=/tmp/tmp.GxeR7KuJdZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.n8Fv35FBto
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("259a5ab9-853d-44e3-b695-34f024ae01eb") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.GxeR7KuJdZ
+ rm /tmp/tmp.n8Fv35FBto /tmp/tmp.GxeR7KuJdZ
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ echo .svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C3pZ7lNFI8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vl9eSvu3v0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C3pZ7lNFI8
++ cat /tmp/tmp.vl9eSvu3v0
++ rm /tmp/tmp.C3pZ7lNFI8 /tmp/tmp.vl9eSvu3v0
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.NZ9CgNexMg
++ mktemp
+ local LAST_ERR=/tmp/tmp.QR8yKUKBod
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NZ9CgNexMg
+ cat /tmp/tmp.QR8yKUKBod
+ rm /tmp/tmp.NZ9CgNexMg /tmp/tmp.QR8yKUKBod
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.EW2C58og8z/find-sharded
+ log 'waiting 60 seconds for stable timestamp in wiredtiger'
+ set +o xtrace
[2025-06-11T12:11:48+0000] waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ log 'running backups'
+ set +o xtrace
[2025-06-11T12:13:08+0000] running backups
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/'
+ kubectl_bin apply -f -
+ yq '.spec.type="incremental-base"'
+ /usr/bin/sed -e 's/storageName:/storageName: aws-s3/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NCPAVBPD2Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.jLwCFvUU2r
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.NCPAVBPD2Z
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.jLwCFvUU2r
+ rm /tmp/tmp.NCPAVBPD2Z /tmp/tmp.jLwCFvUU2r
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq '.spec.type="incremental-base"'
+ /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/'
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vbqDj25hrv
++ mktemp
+ local LAST_ERR=/tmp/tmp.1VEvfZRZPa
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.vbqDj25hrv
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.1VEvfZRZPa
+ rm /tmp/tmp.vbqDj25hrv /tmp/tmp.1VEvfZRZPa
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/'
+ /usr/bin/sed -e 's/storageName:/storageName: azure-blob/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xlu98zFOXf
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ihq046lfQI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.xlu98zFOXf
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.Ihq046lfQI
+ rm /tmp/tmp.xlu98zFOXf /tmp/tmp.Ihq046lfQI
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-aws-s3-sharded to reach ready state...........
+ check_backup_in_storage backup-aws-s3-sharded s3 rs0
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SJ43fj6bfT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l1b68fGrRl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SJ43fj6bfT
++ cat /tmp/tmp.l1b68fGrRl
++ rm /tmp/tmp.SJ43fj6bfT /tmp/tmp.l1b68fGrRl
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/rs0/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:13:39+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/rs0/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10067    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: Ld2ohh+JFLYDt/xUOmLx2WhAVZRW0lZYemOC0JThou/3axwE0BJMZsbImKFeuulQj86rF0eH3OE=
x-amz-request-id: WJZB5YC9C9DS6TZ4
Date: Wed, 11 Jun 2025 12:13:40 GMT
Last-Modified: Wed, 11 Jun 2025 12:13:26 GMT
x-amz-expiration: expiry-date="Fri, 13 Jun 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "20abb56630d518305eed2be8f4565016"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: binary/octet-stream
Content-Length: 10067
Server: AmazonS3

+ check_backup_in_storage backup-aws-s3-sharded s3 cfg
+ local backup=backup-aws-s3-sharded
+ local storage_type=s3
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=s3.amazonaws.com
++ get_backup_dest backup-aws-s3-sharded
++ local backup_name=backup-aws-s3-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed -e 's/.json$//'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.20BDjqI07t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jsOlBqAXKc
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-aws-s3-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.20BDjqI07t
++ cat /tmp/tmp.jsOlBqAXKc
++ rm /tmp/tmp.20BDjqI07t /tmp/tmp.jsOlBqAXKc
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z
+ [[ s3 == \m\i\n\i\o ]]
+ local url=https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/cfg/filelist.pbm
+ log 'checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:13:40+0000] checking if https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/cfg/filelist.pbm exists
+ curl --fail --head https://s3.amazonaws.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 18299    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
x-amz-id-2: SOfTFeYx6cefra4l1CXRRMFRxnWxvHLUUWjmA3AQDVAOBMKPa3M63K0b0HpXS5oQRjqC3cCrZL7VYJOII4zHjfHoNA5NnYKBHEOm0TCTOjE=
x-amz-request-id: E3VE83733BDGE10G
Date: Wed, 11 Jun 2025 12:13:41 GMT
Last-Modified: Wed, 11 Jun 2025 12:13:33 GMT
x-amz-expiration: expiry-date="Fri, 13 Jun 2025 00:00:00 GMT", rule-id="1 Days Cleanup"
ETag: "70d78f38e83ab33e811763cd7d1951fe"
x-amz-server-side-encryption: AES256
Accept-Ranges: bytes
Content-Type: binary/octet-stream
Content-Length: 18299
Server: AmazonS3

+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-gcp-cs-sharded to reach ready state.............
+ check_backup_in_storage backup-gcp-cs-sharded gcs rs0
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
+++ mktemp
++ sed 's|azure://||'
++ local LAST_OUT=/tmp/tmp.aOxMDRaf0V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ADcQn9k8ov
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aOxMDRaf0V
++ cat /tmp/tmp.ADcQn9k8ov
++ rm /tmp/tmp.aOxMDRaf0V /tmp/tmp.ADcQn9k8ov
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/rs0/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:14:08+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/rs0/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10067    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 10067    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: ABgVH89P9huA8N32ZnJBQgiYR8fUAHcOoCbMzEJBArEsfk7fLmVXtnqml2EcbFC5zpdqVnhA
expires: Wed, 11 Jun 2025 13:14:09 GMT
date: Wed, 11 Jun 2025 12:14:09 GMT
cache-control: public, max-age=3600
last-modified: Wed, 11 Jun 2025 12:13:54 GMT
etag: "0967e936ee54821bc65e0d7a6a2adb10"
x-goog-generation: 1749644034130105
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 10067
x-goog-hash: crc32c=Mu84Hg==
x-goog-hash: md5=CWfpNu5UghvGXg16airbEA==
x-goog-expiration: Thu, 12 Jun 2025 12:13:54 GMT
x-goog-storage-class: STANDARD
accept-ranges: bytes
content-length: 10067
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ check_backup_in_storage backup-gcp-cs-sharded gcs cfg
+ local backup=backup-gcp-cs-sharded
+ local storage_type=gcs
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=storage.googleapis.com
++ get_backup_dest backup-gcp-cs-sharded
++ local backup_name=backup-gcp-cs-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ sed 's|azure://||'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3pgkgB8mi5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qTcYWzs6ZJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-gcp-cs-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3pgkgB8mi5
++ cat /tmp/tmp.qTcYWzs6ZJ
++ rm /tmp/tmp.3pgkgB8mi5 /tmp/tmp.qTcYWzs6ZJ
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z
+ [[ gcs == \m\i\n\i\o ]]
+ local url=https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/cfg/filelist.pbm
+ log 'checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:14:10+0000] checking if https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/cfg/filelist.pbm exists
+ curl --fail --head https://storage.googleapis.com/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 19099    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/2 200 
content-type: application/octet-stream
x-guploader-uploadid: ABgVH88Kf2XNfnn3df9aazvdZ5bZuqECgpdj-JTeQpwxxyeKcfu1ztOrrhWMjOm_tILqs_CI
expires: Wed, 11 Jun 2025 13:14:10 GMT
date: Wed, 11 Jun 2025 12:14:10 GMT
cache-control: public, max-age=3600
last-modified: Wed, 11 Jun 2025 12:13:59 GMT
etag: "5a4365993cbc980f06bf5cbe29bfc234"
x-goog-generation: 1749644039443825
x-goog-metageneration: 1
x-goog-stored-content-encoding: identity
x-goog-stored-content-length: 19099
x-goog-hash: crc32c=hyCaRg==
x-goog-hash: md5=WkNlmTy8mA8Gv1y+Kb/CNA==
x-goog-expiration: Thu, 12 Jun 2025 12:13:59 GMT
x-goog-storage-class: STANDARD
accept-ranges: bytes
content-length: 19099
server: UploadServer
alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-azure-blob-sharded to reach ready state............
+ check_backup_in_storage backup-azure-blob-sharded azure rs0
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed 's|azure://||'
++ sed -e 's/.json$//'
++ sed 's|s3://||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DhF3tB948r
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DOEx1qiVua
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.DhF3tB948r
++ cat /tmp/tmp.DOEx1qiVua
++ rm /tmp/tmp.DhF3tB948r /tmp/tmp.DOEx1qiVua
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/rs0/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/rs0/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:14:36+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/rs0/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/rs0/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 12897    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 12897
Content-Type: application/octet-stream
Content-MD5: zrtvC0yvNUprwwkVwgH9Vw==
Last-Modified: Wed, 11 Jun 2025 12:14:25 GMT
ETag: 0x8DDA8E181F00F09
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: 4f1faea2-101e-0048-7eca-dac258000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Wed, 11 Jun 2025 12:14:36 GMT

+ check_backup_in_storage backup-azure-blob-sharded azure cfg
+ local backup=backup-azure-blob-sharded
+ local storage_type=azure
+ local replset=cfg
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=engk8soperators.blob.core.windows.net
++ get_backup_dest backup-azure-blob-sharded
++ local backup_name=backup-azure-blob-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed 's|azure://||'
++ sed 's|s3://||'
++ kubectl_bin get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ sed -e 's/.json$//'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jaiAecq3Jj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O9A90YRYBo
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-azure-blob-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.jaiAecq3Jj
++ cat /tmp/tmp.O9A90YRYBo
++ rm /tmp/tmp.jaiAecq3Jj /tmp/tmp.O9A90YRYBo
++ return 0
+ backup_dest=operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z
+ [[ azure == \m\i\n\i\o ]]
+ local url=https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/cfg/filelist.pbm
+ log 'checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/cfg/filelist.pbm exists'
+ set +o xtrace
[2025-06-11T12:14:38+0000] checking if https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/cfg/filelist.pbm exists
+ curl --fail --head https://engk8soperators.blob.core.windows.net/operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z/cfg/filelist.pbm
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0 19899    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
HTTP/1.1 200 OK
Content-Length: 19899
Content-Type: application/octet-stream
Content-MD5: aQLxpWLg3Y0clZ6RftTFsA==
Last-Modified: Wed, 11 Jun 2025 12:14:28 GMT
ETag: 0x8DDA8E1836FBAC9
Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0
x-ms-request-id: bcc42bee-901e-0046-3aca-daebe8000000
x-ms-version: 2009-09-19
x-ms-lease-status: unlocked
x-ms-blob-type: BlockBlob
Date: Wed, 11 Jun 2025 12:14:37 GMT

+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ local base=true
+ local backup_type=incremental
+ [[ true == \t\r\u\e ]]
+ backup_type=incremental-base
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-minio-sharded/'
+ yq '.spec.type="incremental-base"'
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's/storageName:/storageName: minio/'
+ local LAST_OUT=/tmp/tmp.gwlLBuqsm3
++ mktemp
+ local LAST_ERR=/tmp/tmp.1VDswEgHjt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gwlLBuqsm3
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.1VDswEgHjt
+ rm /tmp/tmp.gwlLBuqsm3 /tmp/tmp.1VDswEgHjt
+ return 0
+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-sharded to reach ready state..........
+ check_backup_in_storage backup-minio-sharded minio rs0
+ local backup=backup-minio-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed 's|azure://||'
++ sed 's|s3://||'
+++ mktemp
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ sed -e 's/.json$//'
++ local LAST_OUT=/tmp/tmp.VaKsjzLBGA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4gKF1yXzKa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VaKsjzLBGA
++ cat /tmp/tmp.4gKF1yXzKa
++ rm /tmp/tmp.VaKsjzLBGA /tmp/tmp.4gKF1yXzKa
++ return 0
+ backup_dest=operator-testing/2025-06-11T12:14:40Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:14:40Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.q58FtKR6YV
++ mktemp
+ local LAST_ERR=/tmp/tmp.c6K3IFgTYH
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:14:40Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.q58FtKR6YV
+ cat /tmp/tmp.c6K3IFgTYH
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.q58FtKR6YV /tmp/tmp.c6K3IFgTYH
+ return 0
+ sleep 1
+ let retry+=1
+ '[' 11 -ge 60 ']'
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:14:40Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.s8ckXkc2Wn
++ mktemp
+ local LAST_ERR=/tmp/tmp.NlODgPOQp7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:14:40Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.s8ckXkc2Wn
+ cat /tmp/tmp.NlODgPOQp7
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-608
+ rm /tmp/tmp.s8ckXkc2Wn /tmp/tmp.NlODgPOQp7
+ return 0
2025-06-11 12:14:58      11959 filelist.pbm
+ run_mongos 'use myApp\n db.test.insert({ x: 100502 })' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.insert({ x: 100502 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RBgxH7GLei
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RzQSeIpwrt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RBgxH7GLei
++ cat /tmp/tmp.RzQSeIpwrt
++ rm /tmp/tmp.RBgxH7GLei /tmp/tmp.RzQSeIpwrt
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TMcKeVz7oC
++ mktemp
+ local LAST_ERR=/tmp/tmp.QW0A2BL4yZ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100502 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TMcKeVz7oC
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("207a8175-a704-4311-8cf7-613ef1469ca1") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.QW0A2BL4yZ
+ rm /tmp/tmp.TMcKeVz7oC /tmp/tmp.QW0A2BL4yZ
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -not-base-sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-not-base-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7pCuDKal5j
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MueCFxCWRr
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7pCuDKal5j
++ cat /tmp/tmp.MueCFxCWRr
++ rm /tmp/tmp.7pCuDKal5j /tmp/tmp.MueCFxCWRr
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.h61cpTBbwx
++ mktemp
+ local LAST_ERR=/tmp/tmp.mCiJVLOvCP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.h61cpTBbwx
+ cat /tmp/tmp.mCiJVLOvCP
+ rm /tmp/tmp.h61cpTBbwx /tmp/tmp.mCiJVLOvCP
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-not-base-sharded.json /tmp/tmp.EW2C58og8z/find-not-base-sharded
+ backup_name_minio_not_base=backup-minio-not-base-sharded
+ run_backup minio backup-minio-not-base-sharded false
+ local storage=minio
+ local backup_name=backup-minio-not-base-sharded
+ local base=false
+ local backup_type=incremental
+ [[ false == \t\r\u\e ]]
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: minio/'
+ yq '.spec.type="incremental"'
+ /usr/bin/sed -e 's/name:/name: backup-minio-not-base-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fJ6c9T9Wj2
++ mktemp
+ local LAST_ERR=/tmp/tmp.obqCM2enzQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fJ6c9T9Wj2
perconaservermongodbbackup.psmdb.percona.com/backup-minio-not-base-sharded created
+ cat /tmp/tmp.obqCM2enzQ
+ rm /tmp/tmp.fJ6c9T9Wj2 /tmp/tmp.obqCM2enzQ
+ return 0
+ wait_backup backup-minio-not-base-sharded
+ local backup_name=backup-minio-not-base-sharded
+ local target_state=ready
+ set +o xtrace
waiting for backup-minio-not-base-sharded to reach ready state.....
+ check_backup_in_storage backup-minio-not-base-sharded minio rs0
+ local backup=backup-minio-not-base-sharded
+ local storage_type=minio
+ local replset=rs0
+ local file=filelist.pbm
+ local endpoint
+ case ${storage_type} in
+ endpoint=minio-service
++ get_backup_dest backup-minio-not-base-sharded
++ local backup_name=backup-minio-not-base-sharded
++ /usr/bin/sed 's|https://engk8soperators.blob.core.windows.net/||'
++ kubectl_bin get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ sed 's|azure://||'
++ sed 's|s3://||'
++ sed -e 's/.json$//'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qNBZ5L3xYU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IngKv08kvU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-not-base-sharded -o 'jsonpath={.status.destination}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qNBZ5L3xYU
++ cat /tmp/tmp.IngKv08kvU
++ rm /tmp/tmp.qNBZ5L3xYU /tmp/tmp.IngKv08kvU
++ return 0
+ backup_dest=operator-testing/2025-06-11T12:15:32Z
+ [[ minio == \m\i\n\i\o ]]
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:15:32Z/rs0/filelist.pbm
+ grep filelist.pbm
++ mktemp
+ local LAST_OUT=/tmp/tmp.ioPtDsycGD
++ mktemp
+ local LAST_ERR=/tmp/tmp.5QOwCG1zgG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/2025-06-11T12:15:32Z/rs0/filelist.pbm
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ioPtDsycGD
+ cat /tmp/tmp.5QOwCG1zgG
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-incremental-sharded-608
+ rm /tmp/tmp.ioPtDsycGD /tmp/tmp.5QOwCG1zgG
+ return 0
2025-06-11 12:15:38      12097 filelist.pbm
+ '[' -z '' ']'
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-06-11T12:15:49+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JBwMu9gjyA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O7uXzeFyGz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JBwMu9gjyA
++ cat /tmp/tmp.O7uXzeFyGz
++ rm /tmp/tmp.JBwMu9gjyA /tmp/tmp.O7uXzeFyGz
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.FLWVtJZ9nW
++ mktemp
+ local LAST_ERR=/tmp/tmp.A2TmtX7vpk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.FLWVtJZ9nW
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("6f7ea322-6569-438f-a826-603fe6064d64") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.A2TmtX7vpk
+ rm /tmp/tmp.FLWVtJZ9nW /tmp/tmp.A2TmtX7vpk
+ return 0
+ log 'check backup and restore -- backup-aws-s3-sharded'
+ set +o xtrace
[2025-06-11T12:15:52+0000] check backup and restore -- backup-aws-s3-sharded
+ /usr/bin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fzgfekrZm4
++ mktemp
+ local LAST_ERR=/tmp/tmp.AFMIKz7zav
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fzgfekrZm4
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.AFMIKz7zav
+ rm /tmp/tmp.fzgfekrZm4 /tmp/tmp.AFMIKz7zav
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-608", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ local LAST_OUT=/tmp/tmp.HIDQHUwfo7
++ mktemp
+ local LAST_ERR=/tmp/tmp.aB3ef1SKEI
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HIDQHUwfo7
+ cat /tmp/tmp.aB3ef1SKEI
+ rm /tmp/tmp.HIDQHUwfo7 /tmp/tmp.aB3ef1SKEI
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ bc -l
++ echo '1.30 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-aws-s3-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-aws-s3-sharded to reach state "ready" .....OK after 5 minutes
+ [[ 0 -eq 1 ]]
++ yq '.metadata.annotations."percona.com/resync-pbm"'
++ kubectl_bin get psmdb some-name -o yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kkUeoOwtnd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7rdV1NYzHK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kkUeoOwtnd
++ cat /tmp/tmp.7rdV1NYzHK
++ rm /tmp/tmp.kkUeoOwtnd /tmp/tmp.7rdV1NYzHK
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Nxx1mcw387
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U4TR6IA0MS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Nxx1mcw387
++ cat /tmp/tmp.U4TR6IA0MS
++ rm /tmp/tmp.Nxx1mcw387 /tmp/tmp.U4TR6IA0MS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.guVB3U7WXt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.PBT0FQ1sh4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.guVB3U7WXt
++ cat /tmp/tmp.PBT0FQ1sh4
++ rm /tmp/tmp.guVB3U7WXt /tmp/tmp.PBT0FQ1sh4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fqmjKIGuBa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0qGknYCHwO
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.fqmjKIGuBa
++ cat /tmp/tmp.0qGknYCHwO
++ rm /tmp/tmp.fqmjKIGuBa /tmp/tmp.0qGknYCHwO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.82ldyIswdg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5DZjfuhmLF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.82ldyIswdg
++ cat /tmp/tmp.5DZjfuhmLF
++ rm /tmp/tmp.82ldyIswdg /tmp/tmp.5DZjfuhmLF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.szfvKMW7WU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DI69ddoxls
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.szfvKMW7WU
++ cat /tmp/tmp.DI69ddoxls
++ rm /tmp/tmp.szfvKMW7WU /tmp/tmp.DI69ddoxls
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ sleep 5
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ryNNhu15OH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rno7hxDFFD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ryNNhu15OH
++ cat /tmp/tmp.Rno7hxDFFD
++ rm /tmp/tmp.ryNNhu15OH /tmp/tmp.Rno7hxDFFD
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TtiFLkC53r
++ mktemp
+ local LAST_ERR=/tmp/tmp.j4urZVSQoy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TtiFLkC53r
+ cat /tmp/tmp.j4urZVSQoy
+ rm /tmp/tmp.TtiFLkC53r /tmp/tmp.j4urZVSQoy
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.EW2C58og8z/find-sharded
+ check_exported_mongos_service_endpoint 34.30.209.98
+ local host=34.30.209.98
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HMsI8aIsJj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UEjNpFF95q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HMsI8aIsJj
++ cat /tmp/tmp.UEjNpFF95q
++ rm /tmp/tmp.HMsI8aIsJj /tmp/tmp.UEjNpFF95q
++ return 0
+ '[' 34.30.209.98 '!=' 34.30.209.98 ']'
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-06-11T12:27:17+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.n6tEk8nQn7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7B18LVfmIa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.n6tEk8nQn7
++ cat /tmp/tmp.7B18LVfmIa
++ rm /tmp/tmp.n6tEk8nQn7 /tmp/tmp.7B18LVfmIa
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9EnYUs6Lgj
++ mktemp
+ local LAST_ERR=/tmp/tmp.xwKW9NRf4e
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9EnYUs6Lgj
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("73ad6f2c-eb4b-4cf5-87ef-7e5ffde8b2ce") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.xwKW9NRf4e
+ rm /tmp/tmp.9EnYUs6Lgj /tmp/tmp.xwKW9NRf4e
+ return 0
+ log 'check backup and restore -- backup-gcp-cs-sharded'
+ set +o xtrace
[2025-06-11T12:27:19+0000] check backup and restore -- backup-gcp-cs-sharded
+ /usr/bin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.GvIDK0U8RA
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZiXtOGF2LA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GvIDK0U8RA
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.ZiXtOGF2LA
+ rm /tmp/tmp.GvIDK0U8RA /tmp/tmp.ZiXtOGF2LA
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
++ mktemp
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-608", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
+ local LAST_OUT=/tmp/tmp.TS6GbAUudg
++ mktemp
+ local LAST_ERR=/tmp/tmp.8p3qh3eiEG
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TS6GbAUudg
+ cat /tmp/tmp.8p3qh3eiEG
+ rm /tmp/tmp.TS6GbAUudg /tmp/tmp.8p3qh3eiEG
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-gcp-cs-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-gcp-cs-sharded to reach state "ready" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SzZMV8yD8x
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gRhjQ5Wi2K
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SzZMV8yD8x
++ cat /tmp/tmp.gRhjQ5Wi2K
++ rm /tmp/tmp.SzZMV8yD8x /tmp/tmp.gRhjQ5Wi2K
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kESCTgNfo0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S2vEyzXXq8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kESCTgNfo0
++ cat /tmp/tmp.S2vEyzXXq8
++ rm /tmp/tmp.kESCTgNfo0 /tmp/tmp.S2vEyzXXq8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tO6LEWD3Uh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pl2r4xeP5W
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tO6LEWD3Uh
++ cat /tmp/tmp.pl2r4xeP5W
++ rm /tmp/tmp.tO6LEWD3Uh /tmp/tmp.pl2r4xeP5W
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eEW0XNdtZL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TkjLZgHbKs
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eEW0XNdtZL
++ cat /tmp/tmp.TkjLZgHbKs
++ rm /tmp/tmp.eEW0XNdtZL /tmp/tmp.TkjLZgHbKs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KxQETTlLeb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yd2xUep1zp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KxQETTlLeb
++ cat /tmp/tmp.yd2xUep1zp
++ rm /tmp/tmp.KxQETTlLeb /tmp/tmp.yd2xUep1zp
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pe5W22EXsm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Fx96SExgXR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pe5W22EXsm
++ cat /tmp/tmp.Fx96SExgXR
++ rm /tmp/tmp.pe5W22EXsm /tmp/tmp.Fx96SExgXR
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ sleep 5
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LnxZk5W7Bm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pP73hByOkL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.LnxZk5W7Bm
++ cat /tmp/tmp.pP73hByOkL
++ rm /tmp/tmp.LnxZk5W7Bm /tmp/tmp.pP73hByOkL
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DJhS7C9ezC
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZzX1snnaAj
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DJhS7C9ezC
+ cat /tmp/tmp.ZzX1snnaAj
+ rm /tmp/tmp.DJhS7C9ezC /tmp/tmp.ZzX1snnaAj
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.EW2C58og8z/find-sharded
+ check_exported_mongos_service_endpoint 34.30.209.98
+ local host=34.30.209.98
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.F6imX8o3Fp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Xf6sMPqwQW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.F6imX8o3Fp
++ cat /tmp/tmp.Xf6sMPqwQW
++ rm /tmp/tmp.F6imX8o3Fp /tmp/tmp.Xf6sMPqwQW
++ return 0
+ '[' 34.30.209.98 '!=' 34.30.209.98 ']'
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-06-11T12:38:25+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WKO41rNuGt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hSdMrJ7coN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WKO41rNuGt
++ cat /tmp/tmp.hSdMrJ7coN
++ rm /tmp/tmp.WKO41rNuGt /tmp/tmp.hSdMrJ7coN
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yKDa8bhhH4
++ mktemp
+ local LAST_ERR=/tmp/tmp.FjP7YLF2OV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yKDa8bhhH4
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("1d620ea6-0aef-4430-adb2-8d05d4f08f43") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.FjP7YLF2OV
+ rm /tmp/tmp.yKDa8bhhH4 /tmp/tmp.FjP7YLF2OV
+ return 0
+ log 'check backup and restore -- backup-azure-blob-sharded'
+ set +o xtrace
[2025-06-11T12:38:29+0000] check backup and restore -- backup-azure-blob-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kTwPOItllU
++ mktemp
+ local LAST_ERR=/tmp/tmp.gJdqepqcRD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kTwPOItllU
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.gJdqepqcRD
+ rm /tmp/tmp.kTwPOItllU /tmp/tmp.gJdqepqcRD
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-608", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fljqTaOo8d
++ mktemp
+ local LAST_ERR=/tmp/tmp.oedfwllvpQ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fljqTaOo8d
+ cat /tmp/tmp.oedfwllvpQ
+ rm /tmp/tmp.fljqTaOo8d /tmp/tmp.oedfwllvpQ
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-azure-blob-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-azure-blob-sharded to reach state "ready" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tXsIPUyiCp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.317VNVw7Sp
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tXsIPUyiCp
++ cat /tmp/tmp.317VNVw7Sp
++ rm /tmp/tmp.tXsIPUyiCp /tmp/tmp.317VNVw7Sp
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c8BA2Fqmqg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vWzG4gE3TD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.c8BA2Fqmqg
++ cat /tmp/tmp.vWzG4gE3TD
++ rm /tmp/tmp.c8BA2Fqmqg /tmp/tmp.vWzG4gE3TD
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rHbYhtA3l4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gTurhKjh8M
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.rHbYhtA3l4
++ cat /tmp/tmp.gTurhKjh8M
++ rm /tmp/tmp.rHbYhtA3l4 /tmp/tmp.gTurhKjh8M
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vdEfWY1sZ9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QCVSXCB2D6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vdEfWY1sZ9
++ cat /tmp/tmp.QCVSXCB2D6
++ rm /tmp/tmp.vdEfWY1sZ9 /tmp/tmp.QCVSXCB2D6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hlLKtoMLXv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LjZXOWiaoE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.hlLKtoMLXv
++ cat /tmp/tmp.LjZXOWiaoE
++ rm /tmp/tmp.hlLKtoMLXv /tmp/tmp.LjZXOWiaoE
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BRRipzKXAa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.afkHbVM3ys
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.BRRipzKXAa
++ cat /tmp/tmp.afkHbVM3ys
++ rm /tmp/tmp.BRRipzKXAa /tmp/tmp.afkHbVM3ys
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ sleep 5
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NVWjtGJFnN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VFcjfRqeH6
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NVWjtGJFnN
++ cat /tmp/tmp.VFcjfRqeH6
++ rm /tmp/tmp.NVWjtGJFnN /tmp/tmp.VFcjfRqeH6
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lZC9slWMDk
++ mktemp
+ local LAST_ERR=/tmp/tmp.qA0lqVQbdO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lZC9slWMDk
+ cat /tmp/tmp.qA0lqVQbdO
+ rm /tmp/tmp.lZC9slWMDk /tmp/tmp.qA0lqVQbdO
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.EW2C58og8z/find-sharded
+ check_exported_mongos_service_endpoint 34.30.209.98
+ local host=34.30.209.98
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.P0KXeExhJe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7pJMGwGBDx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.P0KXeExhJe
++ cat /tmp/tmp.7pJMGwGBDx
++ rm /tmp/tmp.P0KXeExhJe /tmp/tmp.7pJMGwGBDx
++ return 0
+ '[' 34.30.209.98 '!=' 34.30.209.98 ']'
+ run_restore backup-minio-not-base-sharded _restore_sharded
+ local backup_name=backup-minio-not-base-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-06-11T12:49:25+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SQzvo6PXXX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fzi6III9WF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SQzvo6PXXX
++ cat /tmp/tmp.fzi6III9WF
++ rm /tmp/tmp.SQzvo6PXXX /tmp/tmp.fzi6III9WF
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.F47YGlIo8U
++ mktemp
+ local LAST_ERR=/tmp/tmp.k6waloEQ4n
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.F47YGlIo8U
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d50b5409-a3da-48f8-b609-7fe40c760764") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.k6waloEQ4n
+ rm /tmp/tmp.F47YGlIo8U /tmp/tmp.k6waloEQ4n
+ return 0
+ log 'check backup and restore -- backup-minio-not-base-sharded'
+ set +o xtrace
[2025-06-11T12:49:29+0000] check backup and restore -- backup-minio-not-base-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-not-base-sharded/'
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-not-base-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PeVPpuiFoI
++ mktemp
+ local LAST_ERR=/tmp/tmp.NfsY7DoWjm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.PeVPpuiFoI
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-not-base-sharded created
+ cat /tmp/tmp.NfsY7DoWjm
+ rm /tmp/tmp.PeVPpuiFoI /tmp/tmp.NfsY7DoWjm
+ return 0
+ run_recovery_check backup-minio-not-base-sharded _restore_sharded false
+ local backup_name=backup-minio-not-base-sharded
+ local compare_suffix=_restore_sharded
+ local base=false
+ wait_restore backup-minio-not-base-sharded some-name requested 0 3000
+ local backup_name=backup-minio-not-base-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-not-base-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-minio-not-base-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-608", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.CV6qhsrWwD
++ mktemp
+ local LAST_ERR=/tmp/tmp.kzDquRe9ia
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CV6qhsrWwD
+ cat /tmp/tmp.kzDquRe9ia
+ rm /tmp/tmp.CV6qhsrWwD /tmp/tmp.kzDquRe9ia
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ wait_restore backup-minio-not-base-sharded some-name ready 0 3000
+ local backup_name=backup-minio-not-base-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-not-base-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-minio-not-base-sharded to reach state "ready" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
++ kubectl_bin get psmdb some-name -o yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tARhKbCIAM
+++ mktemp
++ yq '.metadata.annotations."percona.com/resync-pbm"'
++ local LAST_ERR=/tmp/tmp.gCNN6pRztN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tARhKbCIAM
++ cat /tmp/tmp.gCNN6pRztN
++ rm /tmp/tmp.tARhKbCIAM /tmp/tmp.gCNN6pRztN
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9JSlpFXoQt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.45zFX2LLIx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9JSlpFXoQt
++ cat /tmp/tmp.45zFX2LLIx
++ rm /tmp/tmp.9JSlpFXoQt /tmp/tmp.45zFX2LLIx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zSyIDDsR6o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZVyr6NL0rg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.zSyIDDsR6o
++ cat /tmp/tmp.ZVyr6NL0rg
++ rm /tmp/tmp.zSyIDDsR6o /tmp/tmp.ZVyr6NL0rg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PVqS95KD4I
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lCioOR8XSY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PVqS95KD4I
++ cat /tmp/tmp.lCioOR8XSY
++ rm /tmp/tmp.PVqS95KD4I /tmp/tmp.lCioOR8XSY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oP0d1KOdYY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7uLTQrHmMV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.oP0d1KOdYY
++ cat /tmp/tmp.7uLTQrHmMV
++ rm /tmp/tmp.oP0d1KOdYY /tmp/tmp.7uLTQrHmMV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.w1kzGkyJUr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WEjnyBZy9Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.w1kzGkyJUr
++ cat /tmp/tmp.WEjnyBZy9Y
++ rm /tmp/tmp.w1kzGkyJUr /tmp/tmp.WEjnyBZy9Y
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ sleep 5
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ [[ false == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -not-base-sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-not-base-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ awk -F: '{print $2}'
++ echo .svc.cluster.local
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EsAdYIqkYm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.21zEyJRgDS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.EsAdYIqkYm
++ cat /tmp/tmp.21zEyJRgDS
++ rm /tmp/tmp.EsAdYIqkYm /tmp/tmp.21zEyJRgDS
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ByDU6S38Pw
++ mktemp
+ local LAST_ERR=/tmp/tmp.QYQ5Rq3N5x
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ByDU6S38Pw
+ cat /tmp/tmp.QYQ5Rq3N5x
+ rm /tmp/tmp.ByDU6S38Pw /tmp/tmp.QYQ5Rq3N5x
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-not-base-sharded.json /tmp/tmp.EW2C58og8z/find-not-base-sharded
+ run_restore backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ log 'drop collection'
+ set +o xtrace
[2025-06-11T12:59:34+0000] drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZgG7plQXYs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KsCrh2Q5V2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ZgG7plQXYs
++ cat /tmp/tmp.KsCrh2Q5V2
++ rm /tmp/tmp.ZgG7plQXYs /tmp/tmp.KsCrh2Q5V2
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.HnXgv3dH8r
++ mktemp
+ local LAST_ERR=/tmp/tmp.dv9BszOTcE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.HnXgv3dH8r
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("082c0c78-19fd-480a-916a-d96277478d53") }
Percona Server for MongoDB server version: v7.0.18-11
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.dv9BszOTcE
+ rm /tmp/tmp.HnXgv3dH8r /tmp/tmp.dv9BszOTcE
+ return 0
+ log 'check backup and restore -- backup-minio-sharded'
+ set +o xtrace
[2025-06-11T12:59:36+0000] check backup and restore -- backup-minio-sharded
+ yq /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/conf/restore.yml
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZR41OqBRDv
++ mktemp
+ local LAST_ERR=/tmp/tmp.iMpF3E9Glv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZR41OqBRDv
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-sharded created
+ cat /tmp/tmp.iMpF3E9Glv
+ rm /tmp/tmp.ZR41OqBRDv /tmp/tmp.iMpF3E9Glv
+ return 0
+ run_recovery_check backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ local compare_suffix=_restore_sharded
+ local base=true
+ wait_restore backup-minio-sharded some-name requested 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-minio-sharded to reach state "requested" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
+ echo

+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("demand-backup-incremental-sharded-608", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gRW90L8W43
++ mktemp
+ local LAST_ERR=/tmp/tmp.H4QpaejLX0
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gRW90L8W43
+ cat /tmp/tmp.H4QpaejLX0
+ rm /tmp/tmp.gRW90L8W43 /tmp/tmp.H4QpaejLX0
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.30 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ [[ 0 -eq 0 ]]
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.EW2C58og8z/statefulset_some-name-rs0.yml
+ wait_restore backup-minio-sharded some-name ready 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ local ok_if_ready=0
+ set +o xtrace
Waiting for the psmdb-restore/restore-backup-minio-sharded object to be createdOK
Waiting psmdb-restore/restore-backup-minio-sharded to reach state "ready" ....OK after 4 minutes
+ [[ 0 -eq 1 ]]
++ yq '.metadata.annotations."percona.com/resync-pbm"'
++ kubectl_bin get psmdb some-name -o yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NGwVEXrCxl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xT6oQeZhz2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NGwVEXrCxl
++ cat /tmp/tmp.xT6oQeZhz2
++ rm /tmp/tmp.NGwVEXrCxl /tmp/tmp.xT6oQeZhz2
++ return 0
+ '[' true == null ']'
+ echo

+ wait_cluster_consistency some-name 60
+ local cluster_name=some-name
+ local wait_time=60
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L0Jdb13IqP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sgfwxKuMMN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.L0Jdb13IqP
++ cat /tmp/tmp.sgfwxKuMMN
++ rm /tmp/tmp.L0Jdb13IqP /tmp/tmp.sgfwxKuMMN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CNhKlDluuY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LWOk9OJ2TI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CNhKlDluuY
++ cat /tmp/tmp.LWOk9OJ2TI
++ rm /tmp/tmp.CNhKlDluuY /tmp/tmp.LWOk9OJ2TI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.t0ugQ04iBc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hIsSqdglFh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.t0ugQ04iBc
++ cat /tmp/tmp.hIsSqdglFh
++ rm /tmp/tmp.t0ugQ04iBc /tmp/tmp.hIsSqdglFh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 60 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aYAejeMq7t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6eI6EbEH6C
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aYAejeMq7t
++ cat /tmp/tmp.6eI6EbEH6C
++ rm /tmp/tmp.aYAejeMq7t /tmp/tmp.6eI6EbEH6C
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo

+ sleep 5
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.
+ [[ true == true ]]
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ local port=27017
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608 mongodb '' '' 27017
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ local mongo_flag=
+ local port=27017
+ local mongo_bin=mongo
++ echo .svc.cluster.local
++ awk -F: '{print $2}'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ suffix_port=
+ [[ -z '' ]]
+ suffix=.svc.cluster.local:27017
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uvm29sA0Gf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.77pfwNWfom
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uvm29sA0Gf
++ cat /tmp/tmp.77pfwNWfom
++ rm /tmp/tmp.uvm29sA0Gf /tmp/tmp.77pfwNWfom
++ return 0
+ local client_container=psmdb-client-b9788d8bc-f6mn9
+ kubectl_bin exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M3gE1EqpLL
++ mktemp
+ local LAST_ERR=/tmp/tmp.nrvCFGLWba
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-b9788d8bc-f6mn9 -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-incremental-sharded-608.svc.cluster.local:27017/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M3gE1EqpLL
+ cat /tmp/tmp.nrvCFGLWba
+ rm /tmp/tmp.M3gE1EqpLL /tmp/tmp.nrvCFGLWba
+ return 0
+ [[ 0 -eq 0 ]]
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/e2e-tests/demand-backup-incremental-sharded/compare/find-sharded.json /tmp/tmp.EW2C58og8z/find-sharded
+ destroy demand-backup-incremental-sharded-608
+ local namespace=demand-backup-incremental-sharded-608
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false ']'
+ delete_backups
+ desc 'Delete psmdb-backup'
+ set +o xtrace


-----------------------------------------------------------------------------------
Delete psmdb-backup
-----------------------------------------------------------------------------------

++ kubectl_bin get psmdb-backup --no-headers
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Xxv7j8ZqQ6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YXnAzdpwfE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ wc -l
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup --no-headers
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Xxv7j8ZqQ6
++ cat /tmp/tmp.YXnAzdpwfE
++ rm /tmp/tmp.Xxv7j8ZqQ6 /tmp/tmp.YXnAzdpwfE
++ return 0
+ '[' 5 '!=' 0 ']'
+ kubectl_bin get psmdb-backup
++ mktemp
+ local LAST_OUT=/tmp/tmp.ED5xJLhzCk
++ mktemp
+ local LAST_ERR=/tmp/tmp.v3qNCA7ft7
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb-backup
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ED5xJLhzCk
NAME                            CLUSTER     STORAGE      DESTINATION                                                                             TYPE               SIZE     STATUS   COMPLETED   AGE
backup-aws-s3-sharded           some-name   aws-s3       s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:11Z      incremental-base   1.72MB   ready    56m         56m
backup-azure-blob-sharded       some-name   azure-blob   azure://operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:14:08Z   incremental-base   3.22MB   ready    55m         56m
backup-gcp-cs-sharded           some-name   gcp-cs       s3://operator-testing/psmdb-demand-backup-incremental-sharded/2025-06-11T12:13:39Z      incremental-base   2.37MB   ready    56m         56m
backup-minio-not-base-sharded   some-name   minio        s3://operator-testing/2025-06-11T12:15:32Z                                              incremental        1.93MB   ready    54m         54m
backup-minio-sharded            some-name   minio        s3://operator-testing/2025-06-11T12:14:40Z                                              incremental-base   3.13MB   ready    55m         55m
+ cat /tmp/tmp.v3qNCA7ft7
+ rm /tmp/tmp.ED5xJLhzCk /tmp/tmp.v3qNCA7ft7
+ return 0
+ kubectl_bin delete psmdb-backup --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.iXx0XMmFjQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8lX64ZQWTg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete psmdb-backup --all
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.iXx0XMmFjQ
perconaservermongodbbackup.psmdb.percona.com "backup-aws-s3-sharded" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-azure-blob-sharded" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-gcp-cs-sharded" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-not-base-sharded" deleted
perconaservermongodbbackup.psmdb.percona.com "backup-minio-sharded" deleted
+ cat /tmp/tmp.8lX64ZQWTg
+ rm /tmp/tmp.iXx0XMmFjQ /tmp/tmp.8lX64ZQWTg
+ return 0
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.BqGTH2W7Uv
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZacDw8KSKP
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BqGTH2W7Uv
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.ZacDw8KSKP
+ rm /tmp/tmp.BqGTH2W7Uv /tmp/tmp.ZacDw8KSKP
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Zq3lxYze48
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dhqemcyszd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Zq3lxYze48
+ cat /tmp/tmp.Dhqemcyszd
+ rm /tmp/tmp.Zq3lxYze48 /tmp/tmp.Dhqemcyszd
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ grep -v NAMESPACE
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.Y1osHd4UWv
++ mktemp
+ local LAST_ERR=/tmp/tmp.NvvttusX2u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Y1osHd4UWv
+ cat /tmp/tmp.NvvttusX2u
+ rm /tmp/tmp.Y1osHd4UWv /tmp/tmp.NvvttusX2u
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.qKBIQXWAYQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kNnxl8Upsx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.qKBIQXWAYQ
+ cat /tmp/tmp.kNnxl8Upsx
+ rm /tmp/tmp.qKBIQXWAYQ /tmp/tmp.kNnxl8Upsx
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.RgiQLjl4qi
++ mktemp
+ local LAST_ERR=/tmp/tmp.QOSTVxQiV8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1926/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.RgiQLjl4qi
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.QOSTVxQiV8
+ rm /tmp/tmp.RgiQLjl4qi /tmp/tmp.QOSTVxQiV8
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.a51lAT8WKF
++ mktemp
+ local LAST_ERR=/tmp/tmp.zJEkf3cHuU
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a51lAT8WKF
+ cat /tmp/tmp.zJEkf3cHuU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a51lAT8WKF
+ cat /tmp/tmp.zJEkf3cHuU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.a51lAT8WKF
+ cat /tmp/tmp.zJEkf3cHuU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.a51lAT8WKF
+ cat /tmp/tmp.zJEkf3cHuU
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.a51lAT8WKF /tmp/tmp.zJEkf3cHuU
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.EW2C58og8z
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-incremental-sharded-608
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.iPZQifQVV6
+ local LAST_OUT=/tmp/tmp.zF2J4U97tZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.1Eb0skopc8
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-incremental-sharded-608
++ mktemp
+ local LAST_ERR=/tmp/tmp.DiYfFtHOXL
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator