Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/logs/demand-backup-physical-sharded.log
WARNING: version difference between client (1.31) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.31) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.31) and server (1.28) exceeds the supported minor version skew of +/-1
+ create_infra demand-backup-physical-sharded-16455
+ local ns=demand-backup-physical-sharded-16455
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace
-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.WvSWFG6J5C
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ip9eLfY1BD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.WvSWFG6J5C
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.Ip9eLfY1BD
+ rm /tmp/tmp.WvSWFG6J5C /tmp/tmp.Ip9eLfY1BD
+ return 0
++ grep -v '\-\-\-'
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27813 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27813 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27813 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-27813 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.GbBo08WyV5
++ mktemp
+ local LAST_ERR=/tmp/tmp.e8MozptWWd
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GbBo08WyV5
+ cat /tmp/tmp.e8MozptWWd
+ rm /tmp/tmp.GbBo08WyV5 /tmp/tmp.e8MozptWWd
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.OFR1yEc6GT
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rb2uQ288XB
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OFR1yEc6GT
+ cat /tmp/tmp.Rb2uQ288XB
+ rm /tmp/tmp.OFR1yEc6GT /tmp/tmp.Rb2uQ288XB
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.TrfwrF43J2
++ mktemp
+ local LAST_ERR=/tmp/tmp.cTDrgm3oUp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TrfwrF43J2
+ cat /tmp/tmp.cTDrgm3oUp
+ rm /tmp/tmp.TrfwrF43J2 /tmp/tmp.cTDrgm3oUp
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.nMSlHWYKRy
++ mktemp
+ local LAST_ERR=/tmp/tmp.00Fi3rjeja
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nMSlHWYKRy
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.00Fi3rjeja
+ rm /tmp/tmp.nMSlHWYKRy /tmp/tmp.00Fi3rjeja
+ return 0
+ check_crd_for_deletion PR-1746-3fa20a85
+ local git_tag=PR-1746-3fa20a85
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1746-3fa20a85/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g'
++ /usr/bin/sed s/---//g
+ for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')'
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gidf5l4geL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QsGQ2N1BBT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.gidf5l4geL
++ cat /tmp/tmp.QsGQ2N1BBT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.gidf5l4geL
++ cat /tmp/tmp.QsGQ2N1BBT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.gidf5l4geL
++ cat /tmp/tmp.QsGQ2N1BBT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.gidf5l4geL
++ cat /tmp/tmp.QsGQ2N1BBT
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.gidf5l4geL /tmp/tmp.QsGQ2N1BBT
++ return 1
+ [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace
-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace
-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ xargs kubectl delete ns
+ set +o xtrace
-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------
++ mktemp
+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.S3xI31i6uz
+ local LAST_OUT=/tmp/tmp.1UCzhqWnty
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.xglKZsQ4Rp
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.QS4RUYqBl6
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.S3xI31i6uz
+ cat /tmp/tmp.xglKZsQ4Rp
+ rm /tmp/tmp.S3xI31i6uz /tmp/tmp.xglKZsQ4Rp
+ return 0
namespace "demand-backup-physical-sharded-27813" deleted
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1UCzhqWnty
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.QS4RUYqBl6
+ rm /tmp/tmp.1UCzhqWnty /tmp/tmp.QS4RUYqBl6
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.G24w8Y15fD
++ mktemp
+ local LAST_ERR=/tmp/tmp.c5NcwH1kjt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.G24w8Y15fD
+ cat /tmp/tmp.c5NcwH1kjt
+ rm /tmp/tmp.G24w8Y15fD /tmp/tmp.c5NcwH1kjt
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace
-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------
+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.SH8qN5Oaaq
++ mktemp
+ local LAST_ERR=/tmp/tmp.wpdaoDdAVk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SH8qN5Oaaq
namespace/psmdb-operator created
+ cat /tmp/tmp.wpdaoDdAVk
+ rm /tmp/tmp.SH8qN5Oaaq /tmp/tmp.wpdaoDdAVk
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kPUTwpdBKW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NYRfJctzAf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kPUTwpdBKW
++ cat /tmp/tmp.NYRfJctzAf
++ rm /tmp/tmp.kPUTwpdBKW /tmp/tmp.NYRfJctzAf
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.svdmmGYTIY
++ mktemp
+ local LAST_ERR=/tmp/tmp.NAunnCsjhy
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.svdmmGYTIY
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5" modified.
+ cat /tmp/tmp.NAunnCsjhy
+ rm /tmp/tmp.svdmmGYTIY /tmp/tmp.NAunnCsjhy
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator'
+ set +o xtrace
-----------------------------------------------------------------------------------
start PSMDB operator
-----------------------------------------------------------------------------------
+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7fSinmThKv
++ mktemp
+ local LAST_ERR=/tmp/tmp.2j6lKKDbNE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.7fSinmThKv
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.2j6lKKDbNE
+ rm /tmp/tmp.7fSinmThKv /tmp/tmp.2j6lKKDbNE
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.8JGfJkZ19Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.79z0EN1XlJ
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.8JGfJkZ19Q
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.79z0EN1XlJ
+ rm /tmp/tmp.8JGfJkZ19Q /tmp/tmp.79z0EN1XlJ
+ return 0
+ yq eval '
(.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1746-3fa20a85") |
((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1YcgXtsCo9
++ mktemp
+ local LAST_ERR=/tmp/tmp.43tMg0PHZq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1YcgXtsCo9
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.43tMg0PHZq
+ rm /tmp/tmp.1YcgXtsCo9 /tmp/tmp.43tMg0PHZq
+ return 0
+ sleep 2
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rk3TbM6CD1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7VIFsDgW5Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Rk3TbM6CD1
++ cat /tmp/tmp.7VIFsDgW5Y
++ rm /tmp/tmp.Rk3TbM6CD1 /tmp/tmp.7VIFsDgW5Y
++ return 0
+ wait_pod percona-server-mongodb-operator-556cd9cf9-2xkxb
+ local pod=percona-server-mongodb-operator-556cd9cf9-2xkxb
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-556cd9cf9-2xkxb to be ready.OK
+ create_namespace demand-backup-physical-sharded-16455
+ local namespace=demand-backup-physical-sharded-16455
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace
-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace
-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------
+ egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-physical-sharded-16455'
+ set +o xtrace
-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-sharded-16455
-----------------------------------------------------------------------------------
+ kubectl_bin delete namespace demand-backup-physical-sharded-16455 --ignore-not-found
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vwrr5nc1A3
++ mktemp
+ xargs kubectl delete ns
+ local LAST_ERR=/tmp/tmp.t5id6DJoqK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.T8FyRtM7XU
++ mktemp
+ local LAST_ERR=/tmp/tmp.0fDLFRz6R1
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-physical-sharded-16455 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vwrr5nc1A3
+ cat /tmp/tmp.t5id6DJoqK
+ rm /tmp/tmp.Vwrr5nc1A3 /tmp/tmp.t5id6DJoqK
+ return 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.T8FyRtM7XU
+ cat /tmp/tmp.0fDLFRz6R1
+ rm /tmp/tmp.T8FyRtM7XU /tmp/tmp.0fDLFRz6R1
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-sharded-16455
++ mktemp
+ local LAST_OUT=/tmp/tmp.XeOV6sOZsU
++ mktemp
+ local LAST_ERR=/tmp/tmp.wvNM8oNEzi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-sharded-16455
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.XeOV6sOZsU
+ cat /tmp/tmp.wvNM8oNEzi
+ rm /tmp/tmp.XeOV6sOZsU /tmp/tmp.wvNM8oNEzi
+ return 0
+ desc 'create namespace demand-backup-physical-sharded-16455'
+ set +o xtrace
-----------------------------------------------------------------------------------
create namespace demand-backup-physical-sharded-16455
-----------------------------------------------------------------------------------
+ kubectl_bin create namespace demand-backup-physical-sharded-16455
++ mktemp
+ local LAST_OUT=/tmp/tmp.B62YVu21xQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kHJkMpwxth
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-physical-sharded-16455
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.B62YVu21xQ
namespace/demand-backup-physical-sharded-16455 created
+ cat /tmp/tmp.kHJkMpwxth
+ rm /tmp/tmp.B62YVu21xQ /tmp/tmp.kHJkMpwxth
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WHOkUgiYP8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8uVOfFu0KM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WHOkUgiYP8
++ cat /tmp/tmp.8uVOfFu0KM
++ rm /tmp/tmp.WHOkUgiYP8 /tmp/tmp.8uVOfFu0KM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5 --namespace=demand-backup-physical-sharded-16455
++ mktemp
+ local LAST_OUT=/tmp/tmp.6XXEdU1qMm
++ mktemp
+ local LAST_ERR=/tmp/tmp.wGrWzAnfDS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5 --namespace=demand-backup-physical-sharded-16455
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.6XXEdU1qMm
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1746-3fa20a85-1-cluster5" modified.
+ cat /tmp/tmp.wGrWzAnfDS
+ rm /tmp/tmp.6XXEdU1qMm /tmp/tmp.wGrWzAnfDS
+ return 0
+ deploy_minio
+ desc 'install Minio'
+ set +o xtrace
-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------
+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ helm repo remove minio
"minio" has been removed from your repositories
+ helm repo add minio https://charts.min.io/
"minio" has been added to your repositories
+ retry 10 60 helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.0.14 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio
NAME: minio-service
LAST DEPLOYED: Mon Dec 2 22:05:29 2024
NAMESPACE: demand-backup-physical-sharded-16455
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-physical-sharded-16455.svc.cluster.local
To access MinIO from localhost, run the below commands:
1. export POD_NAME=$(kubectl get pods --namespace demand-backup-physical-sharded-16455 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")
2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-physical-sharded-16455
Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/
You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:
1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart
2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-physical-sharded-16455 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-physical-sharded-16455 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000
3. mc ls minio-service-local
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ef5dQb461u
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6obMTDDsKh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Ef5dQb461u
++ cat /tmp/tmp.6obMTDDsKh
++ rm /tmp/tmp.Ef5dQb461u /tmp/tmp.6obMTDDsKh
++ return 0
+ MINIO_POD=minio-service-847fc8bb8d-n49pq
+ wait_pod minio-service-847fc8bb8d-n49pq
+ local pod=minio-service-847fc8bb8d-n49pq
+ set +o xtrace
waiting for pod/minio-service-847fc8bb8d-n49pq to be ready.OK
+ '[' -n psmdb-operator ']'
+ kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-16455.svc.cluster.local --tcp=9000
++ mktemp
+ local LAST_OUT=/tmp/tmp.01fbkAYOFD
++ mktemp
+ local LAST_ERR=/tmp/tmp.6tYz2UH4AN
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-physical-sharded-16455.svc.cluster.local --tcp=9000
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.01fbkAYOFD
service/minio-service created
+ cat /tmp/tmp.6tYz2UH4AN
+ rm /tmp/tmp.01fbkAYOFD /tmp/tmp.6tYz2UH4AN
+ return 0
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
/usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
++ mktemp
+ local LAST_OUT=/tmp/tmp.5yMfZUFgMX
++ mktemp
+ local LAST_ERR=/tmp/tmp.SG2xe4Zeql
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
/usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5yMfZUFgMX
make_bucket: operator-testing
pod "aws-cli" deleted
+ cat /tmp/tmp.SG2xe4Zeql
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state
+ rm /tmp/tmp.5yMfZUFgMX /tmp/tmp.SG2xe4Zeql
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace
-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------
+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.OgfIyTgUqB
++ mktemp
+ local LAST_ERR=/tmp/tmp.1wuTYijzPm
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OgfIyTgUqB
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.1wuTYijzPm
+ rm /tmp/tmp.OgfIyTgUqB /tmp/tmp.1wuTYijzPm
+ return 0
+ desc 'Testing on sharded cluster'
+ set +o xtrace
-----------------------------------------------------------------------------------
Testing on sharded cluster
-----------------------------------------------------------------------------------
+ echo 'Creating PSMDB cluster'
Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.UQHjw56Fu1
++ mktemp
+ local LAST_ERR=/tmp/tmp.V6hUqseExh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.UQHjw56Fu1
secret/some-users created
+ cat /tmp/tmp.V6hUqseExh
+ rm /tmp/tmp.UQHjw56Fu1 /tmp/tmp.V6hUqseExh
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-backup-mongodb:2.7.0"'
++ mktemp
+ yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1746-3fa20a85"'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
+ local LAST_OUT=/tmp/tmp.3zBZSTdTsl
+ yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"'
++ mktemp
+ local LAST_ERR=/tmp/tmp.39hUL3UqTL
+ local exit_status=0
+ local timeout=4
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3zBZSTdTsl
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.39hUL3UqTL
+ rm /tmp/tmp.3zBZSTdTsl /tmp/tmp.39hUL3UqTL
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.YVWLB8BoBb
++ mktemp
+ local LAST_ERR=/tmp/tmp.yRvKra122u
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.YVWLB8BoBb
deployment.apps/psmdb-client created
+ cat /tmp/tmp.yRvKra122u
+ rm /tmp/tmp.YVWLB8BoBb /tmp/tmp.yRvKra122u
+ return 0
+ echo 'check if all pods started'
check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready..................OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-rs0-1
+ local pod=some-name-rs0-1
+ set +o xtrace
waiting for pod/some-name-rs0-1 to be ready........OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N2Dev5xSao
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kWbLEJCtnN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.N2Dev5xSao
++ cat /tmp/tmp.kWbLEJCtnN
++ rm /tmp/tmp.N2Dev5xSao /tmp/tmp.kWbLEJCtnN
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-rs0-2
+ local pod=some-name-rs0-2
+ set +o xtrace
waiting for pod/some-name-rs0-2 to be ready......OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xL1q05zb1R
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XB4otwtBRa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.xL1q05zb1R
++ cat /tmp/tmp.XB4otwtBRa
++ rm /tmp/tmp.xL1q05zb1R /tmp/tmp.XB4otwtBRa
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness..............................
+ wait_for_running some-name-cfg 3
+ local name=some-name-cfg
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=cfg
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-cfg-0
+ local pod=some-name-cfg-0
+ set +o xtrace
waiting for pod/some-name-cfg-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-cfg-1
+ local pod=some-name-cfg-1
+ set +o xtrace
waiting for pod/some-name-cfg-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8bANlGRtRp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cgnheZBec3
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8bANlGRtRp
++ cat /tmp/tmp.cgnheZBec3
++ rm /tmp/tmp.8bANlGRtRp /tmp/tmp.cgnheZBec3
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-cfg-2
+ local pod=some-name-cfg-2
+ set +o xtrace
waiting for pod/some-name-cfg-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vhJ1aeFPqp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pXgXyYubPK
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.vhJ1aeFPqp
++ cat /tmp/tmp.pXgXyYubPK
++ rm /tmp/tmp.vhJ1aeFPqp /tmp/tmp.pXgXyYubPK
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_for_running some-name-mongos 3
+ local name=some-name-mongos
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=mongos
+ local cluster_name=some-name
++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-mongos-0
+ local pod=some-name-mongos-0
+ set +o xtrace
waiting for pod/some-name-mongos-0 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 1 -eq 2 ]]
+ wait_pod some-name-mongos-1
+ local pod=some-name-mongos-1
+ set +o xtrace
waiting for pod/some-name-mongos-1 to be ready.OK
+ for i in '$(seq 0 $last_pod)'
+ [[ 2 -eq 2 ]]
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pZPoEKCQ2v
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5Bq54PgljA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pZPoEKCQ2v
++ cat /tmp/tmp.5Bq54PgljA
++ rm /tmp/tmp.pZPoEKCQ2v /tmp/tmp.5Bq54PgljA
++ return 0
+ [[ '' == \t\r\u\e ]]
+ wait_pod some-name-mongos-2
+ local pod=some-name-mongos-2
+ set +o xtrace
waiting for pod/some-name-mongos-2 to be ready.OK
++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.74LWlGGyBY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ay90DTOcwF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.74LWlGGyBY
++ cat /tmp/tmp.ay90DTOcwF
++ rm /tmp/tmp.74LWlGGyBY /tmp/tmp.ay90DTOcwF
++ return 0
+ [[ '' == \t\r\u\e ]]
+ sleep 10
+ [[ true == \t\r\u\e ]]
+ set +x
Waiting for cluster readyness
+ wait_cluster_consistency some-name
+ local cluster_name=some-name
+ local wait_time=32
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.We1kH741Ud
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xEQkyryiWj
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.We1kH741Ud
++ cat /tmp/tmp.xEQkyryiWj
++ rm /tmp/tmp.We1kH741Ud /tmp/tmp.xEQkyryiWj
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo
++ kubectl_bin get svc some-name-mongos '-o=jsonpath={.status}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iLyXslJAmZ
++ jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0] | if .ip then .ip else .hostname end'
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5mvqJhIekE
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get svc some-name-mongos '-o=jsonpath={.status}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.iLyXslJAmZ
++ cat /tmp/tmp.5mvqJhIekE
++ rm /tmp/tmp.iLyXslJAmZ /tmp/tmp.5mvqJhIekE
++ return 0
+ lbEndpoint=146.148.39.140
+ '[' -z 146.148.39.140 ']'
+ run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})'
+ local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dBZhjhy90K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cxqsVTG58a
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.dBZhjhy90K
++ cat /tmp/tmp.cxqsVTG58a
++ rm /tmp/tmp.dBZhjhy90K /tmp/tmp.cxqsVTG58a
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.DkOqkIHKlX
++ mktemp
+ local LAST_ERR=/tmp/tmp.znnpa94eGe
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DkOqkIHKlX
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("d0c238e3-ad14-4023-a09e-055ff9ffd5d9") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
Successfully added user: {
"user" : "myApp",
"roles" : [
{
"db" : "myApp",
"role" : "readWrite"
}
]
}
bye
+ cat /tmp/tmp.znnpa94eGe
+ rm /tmp/tmp.DkOqkIHKlX /tmp/tmp.znnpa94eGe
+ return 0
+ sleep 1
+ run_mongos 'use myApp\n db.test.insert({ x: 100501 })' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=use myApp\n db.test.insert({ x: 100501 })'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aagXcuNchA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0bEWdRNlh1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aagXcuNchA
++ cat /tmp/tmp.0bEWdRNlh1
++ rm /tmp/tmp.aagXcuNchA /tmp/tmp.0bEWdRNlh1
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.yyCyMWswRa
++ mktemp
+ local LAST_ERR=/tmp/tmp.a3cfaqDJch
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100501 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.yyCyMWswRa
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("bb61087f-3d07-47b8-9890-88d1ad21f404") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
WriteResult({ "nInserted" : 1 })
bye
+ cat /tmp/tmp.a3cfaqDJch
+ rm /tmp/tmp.yyCyMWswRa /tmp/tmp.a3cfaqDJch
+ return 0
+ sleep 5
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3GwU7b9tN2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kjDKWBQhXL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.3GwU7b9tN2
++ cat /tmp/tmp.kjDKWBQhXL
++ rm /tmp/tmp.3GwU7b9tN2 /tmp/tmp.kjDKWBQhXL
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.M4zigToalK
++ mktemp
+ local LAST_ERR=/tmp/tmp.ur8K0o4B58
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.M4zigToalK
+ cat /tmp/tmp.ur8K0o4B58
+ rm /tmp/tmp.M4zigToalK /tmp/tmp.ur8K0o4B58
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.hqutgSWKJt/find-sharded
+ echo 'waiting 60 seconds for stable timestamp in wiredtiger'
waiting 60 seconds for stable timestamp in wiredtiger
+ sleep 80
+ echo 'running backups'
running backups
+ backup_name_minio=backup-minio-sharded
+ run_backup minio backup-minio-sharded
+ local storage=minio
+ local backup_name=backup-minio-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: minio/'
+ /usr/bin/sed -e 's/name:/name: backup-minio-sharded/'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.lT9PO7v7FN
++ mktemp
+ local LAST_ERR=/tmp/tmp.qb4KO23S6m
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lT9PO7v7FN
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded created
+ cat /tmp/tmp.qb4KO23S6m
+ rm /tmp/tmp.lT9PO7v7FN /tmp/tmp.qb4KO23S6m
+ return 0
+ '[' -z '' ']'
+ backup_name_aws=backup-aws-s3-sharded
+ backup_name_gcp=backup-gcp-cs-sharded
+ backup_name_azure=backup-azure-blob-sharded
+ run_backup aws-s3 backup-aws-s3-sharded
+ local storage=aws-s3
+ local backup_name=backup-aws-s3-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/storageName:/storageName: aws-s3/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: backup-aws-s3-sharded/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.tqjKaA9IGv
++ mktemp
+ local LAST_ERR=/tmp/tmp.JvtXHPQemw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.tqjKaA9IGv
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded created
+ cat /tmp/tmp.JvtXHPQemw
+ rm /tmp/tmp.tqjKaA9IGv /tmp/tmp.JvtXHPQemw
+ return 0
+ run_backup gcp-cs backup-gcp-cs-sharded
+ local storage=gcp-cs
+ local backup_name=backup-gcp-cs-sharded
+ /usr/bin/sed -e 's/name:/name: backup-gcp-cs-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/storageName:/storageName: gcp-cs/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.V1SX4he7r9
++ mktemp
+ local LAST_ERR=/tmp/tmp.bthFy6ldfg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V1SX4he7r9
perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs-sharded created
+ cat /tmp/tmp.bthFy6ldfg
+ rm /tmp/tmp.V1SX4he7r9 /tmp/tmp.bthFy6ldfg
+ return 0
+ run_backup azure-blob backup-azure-blob-sharded
+ local storage=azure-blob
+ local backup_name=backup-azure-blob-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/backup.yml
+ /usr/bin/sed -e 's/name:/name: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/storageName:/storageName: azure-blob/'
++ mktemp
+ local LAST_OUT=/tmp/tmp.OdpJE9cyd8
++ mktemp
+ local LAST_ERR=/tmp/tmp.3sbAF2TPcD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.OdpJE9cyd8
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded created
+ cat /tmp/tmp.3sbAF2TPcD
+ rm /tmp/tmp.OdpJE9cyd8 /tmp/tmp.3sbAF2TPcD
+ return 0
+ wait_backup backup-aws-s3-sharded
+ local backup_name=backup-aws-s3-sharded
+ set +o xtrace
backup-aws-s3-sharded......................................
+ wait_backup backup-gcp-cs-sharded
+ local backup_name=backup-gcp-cs-sharded
+ set +o xtrace
backup-gcp-cs-sharded.......................
+ wait_backup backup-azure-blob-sharded
+ local backup_name=backup-azure-blob-sharded
+ set +o xtrace
backup-azure-blob-sharded.....................
+ wait_backup backup-minio-sharded
+ local backup_name=backup-minio-sharded
+ set +o xtrace
backup-minio-sharded.
+ '[' -z '' ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cm13xd1ZhU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZkdoVcczwI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cm13xd1ZhU
++ cat /tmp/tmp.ZkdoVcczwI
++ rm /tmp/tmp.cm13xd1ZhU /tmp/tmp.ZkdoVcczwI
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.lQ7fVVEa83
++ mktemp
+ local LAST_ERR=/tmp/tmp.gWphWdLw4L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.lQ7fVVEa83
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("2ddecf4c-09d9-426b-b4a1-c8d1282a45bc") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.gWphWdLw4L
+ rm /tmp/tmp.lQ7fVVEa83 /tmp/tmp.gWphWdLw4L
+ return 0
+ echo 'check backup and restore -- aws-s3'
check backup and restore -- aws-s3
+ run_restore backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ /usr/bin/sed -e 's/name:/name: restore-backup-aws-s3-sharded/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3Q9biLgMOk
++ mktemp
+ local LAST_ERR=/tmp/tmp.7AyF30TsUF
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's/backupName:/backupName: backup-aws-s3-sharded/'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.3Q9biLgMOk
perconaservermongodbrestore.psmdb.percona.com/restore-backup-aws-s3-sharded created
+ cat /tmp/tmp.7AyF30TsUF
+ rm /tmp/tmp.3Q9biLgMOk /tmp/tmp.7AyF30TsUF
+ return 0
+ run_recovery_check backup-aws-s3-sharded _restore_sharded
+ local backup_name=backup-aws-s3-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-aws-s3-sharded some-name requested 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-aws-s3-sharded to reach requested state.................................................................................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo
+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
del(.metadata.ownerReferences[].apiVersion) |
del(.metadata.managedFields) |
del(.. | select(has("creationTimestamp")).creationTimestamp) |
del(.. | select(has("namespace")).namespace) |
del(.. | select(has("uid")).uid) |
del(.metadata.resourceVersion) |
del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
del(.metadata.selfLink) |
del(.metadata.annotations."cloud.google.com/neg") |
del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
del(.. | select(has("image")).image) |
del(.. | select(has("clusterIP")).clusterIP) |
del(.. | select(has("clusterIPs")).clusterIPs) |
del(.. | select(has("dataSource")).dataSource) |
del(.. | select(has("procMount")).procMount) |
del(.. | select(has("storageClassName")).storageClassName) |
del(.. | select(has("finalizers")).finalizers) |
del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
del(.. | select(has("volumeName")).volumeName) |
del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
del(.spec.volumeMode) |
del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
del(.. | select(has("nodePort")).nodePort) |
del(.status) |
(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-16455", "NAME_SPACE") |
del(.spec.volumeClaimTemplates[].apiVersion) |
del(.spec.volumeClaimTemplates[].kind) |
del(.spec.ipFamilies) |
del(.spec.ipFamilyPolicy) |
(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.gBk6Wk9cQO
++ mktemp
+ local LAST_ERR=/tmp/tmp.Iqr2jMtFOz
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.gBk6Wk9cQO
+ cat /tmp/tmp.Iqr2jMtFOz
+ rm /tmp/tmp.gBk6Wk9cQO /tmp/tmp.Iqr2jMtFOz
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ wait_restore backup-aws-s3-sharded some-name ready 0 3000
+ local backup_name=backup-aws-s3-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-aws-s3-sharded to reach ready state........................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.KRUtg2t8BN
++ mktemp
+ local LAST_ERR=/tmp/tmp.M61wGcON7j
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.KRUtg2t8BN
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-16455"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-backup-mongodb:2.7.0","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
percona.com/resync-pbm: "true"
creationTimestamp: "2024-12-02T22:06:23Z"
generation: 2
name: some-name
namespace: demand-backup-physical-sharded-16455
resourceVersion: "35837"
uid: b160147e-1cf6-4530-ac52-8a83191c65d3
spec:
backup:
enabled: true
image: perconalab/percona-backup-mongodb:2.7.0
storages:
aws-s3:
s3:
bucket: operator-testing
credentialsSecret: aws-s3-secret
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
azure-blob:
azure:
container: operator-testing
credentialsSecret: azure-secret
prefix: psmdb-demand-backup-physical-sharded
type: azure
gcp-cs:
s3:
bucket: operator-testing
credentialsSecret: gcp-cs-secret
endpointUrl: https://storage.googleapis.com
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
minio:
s3:
bucket: operator-testing
credentialsSecret: minio-secret
endpointUrl: http://minio-service:9000/
insecureSkipTLSVerify: false
region: us-east-1
type: s3
tasks:
- compressionType: gzip
enabled: true
name: weekly
schedule: 0 0 * * 0
storageName: aws-s3
crVersion: 1.19.0
image: perconalab/percona-server-mongodb-operator:main-mongod7.0
imagePullPolicy: Always
replsets:
- affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
name: rs0
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
secrets:
users: some-users
sharding:
configsvrReplSet:
affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
enabled: true
mongos:
affinity:
antiAffinityTopologyKey: none
expose:
type: LoadBalancer
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
upgradeOptions:
apply: Never
status:
conditions:
- lastTransitionTime: "2024-12-02T22:06:26Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:08:34Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:08:34Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:09:12Z"
reason: MongosReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:14:56Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:15:30Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:15:30Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:15:37Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:15:37Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:16:10Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:16:10Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:16:12Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:16:12Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:16:44Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:16:44Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:16:50Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:16:50Z"
status: "True"
type: initializing
host: 146.148.39.140
mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
mongoVersion: 7.0.15-9
mongos:
ready: 0
size: 0
status: initializing
observedGeneration: 2
ready: 6
replsets:
cfg:
initialized: true
ready: 3
size: 3
status: ready
rs0:
added_as_shard: true
initialized: true
ready: 3
size: 3
status: ready
size: 6
state: initializing
+ cat /tmp/tmp.M61wGcON7j
+ rm /tmp/tmp.KRUtg2t8BN /tmp/tmp.M61wGcON7j
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.urq0iuO79q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xPEHlOMgUF
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.urq0iuO79q
++ cat /tmp/tmp.xPEHlOMgUF
++ rm /tmp/tmp.urq0iuO79q /tmp/tmp.xPEHlOMgUF
++ return 0
+ '[' true == null ']'
+ echo
+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Z0LaDBfkR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EAOK1Olcg4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5Z0LaDBfkR
++ cat /tmp/tmp.EAOK1Olcg4
++ rm /tmp/tmp.5Z0LaDBfkR /tmp/tmp.EAOK1Olcg4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4TENo9cBlv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.m2L0aZjxp4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4TENo9cBlv
++ cat /tmp/tmp.m2L0aZjxp4
++ rm /tmp/tmp.4TENo9cBlv /tmp/tmp.m2L0aZjxp4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6vEhQbRW4S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9afqYk95r1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.6vEhQbRW4S
++ cat /tmp/tmp.9afqYk95r1
++ rm /tmp/tmp.6vEhQbRW4S /tmp/tmp.9afqYk95r1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4euVrq4vNt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yYpWInZgKT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4euVrq4vNt
++ cat /tmp/tmp.yYpWInZgKT
++ rm /tmp/tmp.4euVrq4vNt /tmp/tmp.yYpWInZgKT
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Pka79HpCx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9JrfB26lXx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9Pka79HpCx
++ cat /tmp/tmp.9JrfB26lXx
++ rm /tmp/tmp.9Pka79HpCx /tmp/tmp.9JrfB26lXx
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5PyYx0K2cB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rZ36KDNrn4
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5PyYx0K2cB
++ cat /tmp/tmp.rZ36KDNrn4
++ rm /tmp/tmp.5PyYx0K2cB /tmp/tmp.rZ36KDNrn4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.phKQLVODUo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IVkfdNc00q
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.phKQLVODUo
++ cat /tmp/tmp.IVkfdNc00q
++ rm /tmp/tmp.phKQLVODUo /tmp/tmp.IVkfdNc00q
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9XcpBXI0TY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mbWn00AnRn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9XcpBXI0TY
++ cat /tmp/tmp.mbWn00AnRn
++ rm /tmp/tmp.9XcpBXI0TY /tmp/tmp.mbWn00AnRn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S2H9YYoJXi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.baWH5aS7nN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.S2H9YYoJXi
++ cat /tmp/tmp.baWH5aS7nN
++ rm /tmp/tmp.S2H9YYoJXi /tmp/tmp.baWH5aS7nN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gZVPfMaoaf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.de2RtqK3Ko
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gZVPfMaoaf
++ cat /tmp/tmp.de2RtqK3Ko
++ rm /tmp/tmp.gZVPfMaoaf /tmp/tmp.de2RtqK3Ko
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v28nT9WArw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.johUdASDmi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.v28nT9WArw
++ cat /tmp/tmp.johUdASDmi
++ rm /tmp/tmp.v28nT9WArw /tmp/tmp.johUdASDmi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0WPLeLC2fX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OIvvWBdpSU
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0WPLeLC2fX
++ cat /tmp/tmp.OIvvWBdpSU
++ rm /tmp/tmp.0WPLeLC2fX /tmp/tmp.OIvvWBdpSU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.438V3dOUdL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oWgXcC2meL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.438V3dOUdL
++ cat /tmp/tmp.oWgXcC2meL
++ rm /tmp/tmp.438V3dOUdL /tmp/tmp.oWgXcC2meL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uZT7ZOgoGy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1uhOyYTS5h
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.uZT7ZOgoGy
++ cat /tmp/tmp.1uhOyYTS5h
++ rm /tmp/tmp.uZT7ZOgoGy /tmp/tmp.1uhOyYTS5h
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VDRNDclKAp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rfi3kr4IBd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.VDRNDclKAp
++ cat /tmp/tmp.Rfi3kr4IBd
++ rm /tmp/tmp.VDRNDclKAp /tmp/tmp.Rfi3kr4IBd
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eAXIEwjWEz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0UQZIBW5k8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.eAXIEwjWEz
++ cat /tmp/tmp.0UQZIBW5k8
++ rm /tmp/tmp.eAXIEwjWEz /tmp/tmp.0UQZIBW5k8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JVm5ZRgsey
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KOSwifo2GL
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JVm5ZRgsey
++ cat /tmp/tmp.KOSwifo2GL
++ rm /tmp/tmp.JVm5ZRgsey /tmp/tmp.KOSwifo2GL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 17 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CPzhvbFjuE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xNZMPsexfb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CPzhvbFjuE
++ cat /tmp/tmp.xNZMPsexfb
++ rm /tmp/tmp.CPzhvbFjuE /tmp/tmp.xNZMPsexfb
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish....................................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R5bPKzXUxL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aKBmSQpW2v
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.R5bPKzXUxL
++ cat /tmp/tmp.aKBmSQpW2v
++ rm /tmp/tmp.R5bPKzXUxL /tmp/tmp.aKBmSQpW2v
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.V2foA2szri
++ mktemp
+ local LAST_ERR=/tmp/tmp.Kkm4oknRFr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.V2foA2szri
+ cat /tmp/tmp.Kkm4oknRFr
+ rm /tmp/tmp.V2foA2szri /tmp/tmp.Kkm4oknRFr
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.hqutgSWKJt/find-sharded
+ check_exported_mongos_service_endpoint 146.148.39.140
+ local host=146.148.39.140
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.afSrrkqVAe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.82MuhUQPMb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.afSrrkqVAe
++ cat /tmp/tmp.82MuhUQPMb
++ rm /tmp/tmp.afSrrkqVAe /tmp/tmp.82MuhUQPMb
++ return 0
+ '[' 146.148.39.140 '!=' 146.148.39.140 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cQBBye1zRY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.l6bSwhkDYx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cQBBye1zRY
++ cat /tmp/tmp.l6bSwhkDYx
++ rm /tmp/tmp.cQBBye1zRY /tmp/tmp.l6bSwhkDYx
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.BAgI0sF8Us
++ mktemp
+ local LAST_ERR=/tmp/tmp.oOXYlyy8OE
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BAgI0sF8Us
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("e8098990-ab23-46e4-80b3-baa65cbdd681") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.oOXYlyy8OE
+ rm /tmp/tmp.BAgI0sF8Us /tmp/tmp.oOXYlyy8OE
+ return 0
+ echo 'check backup and restore -- gcp-cs'
check backup and restore -- gcp-cs
+ run_restore backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.jCwTrHlXtB
++ mktemp
+ local LAST_ERR=/tmp/tmp.cqZ7V9gId0
+ local exit_status=0
+ local timeout=4
+ /usr/bin/sed -e 's/name:/name: restore-backup-gcp-cs-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-gcp-cs-sharded/'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jCwTrHlXtB
perconaservermongodbrestore.psmdb.percona.com/restore-backup-gcp-cs-sharded created
+ cat /tmp/tmp.cqZ7V9gId0
+ rm /tmp/tmp.jCwTrHlXtB /tmp/tmp.cqZ7V9gId0
+ return 0
+ run_recovery_check backup-gcp-cs-sharded _restore_sharded
+ local backup_name=backup-gcp-cs-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-gcp-cs-sharded some-name requested 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-gcp-cs-sharded to reach requested state..................................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo
+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
del(.metadata.ownerReferences[].apiVersion) |
del(.metadata.managedFields) |
del(.. | select(has("creationTimestamp")).creationTimestamp) |
del(.. | select(has("namespace")).namespace) |
del(.. | select(has("uid")).uid) |
del(.metadata.resourceVersion) |
del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
del(.metadata.selfLink) |
del(.metadata.annotations."cloud.google.com/neg") |
del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
del(.. | select(has("image")).image) |
del(.. | select(has("clusterIP")).clusterIP) |
del(.. | select(has("clusterIPs")).clusterIPs) |
del(.. | select(has("dataSource")).dataSource) |
del(.. | select(has("procMount")).procMount) |
del(.. | select(has("storageClassName")).storageClassName) |
del(.. | select(has("finalizers")).finalizers) |
del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
del(.. | select(has("volumeName")).volumeName) |
del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
del(.spec.volumeMode) |
del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
del(.. | select(has("nodePort")).nodePort) |
del(.status) |
(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-16455", "NAME_SPACE") |
del(.spec.volumeClaimTemplates[].apiVersion) |
del(.spec.volumeClaimTemplates[].kind) |
del(.spec.ipFamilies) |
del(.spec.ipFamilyPolicy) |
(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.BgDKe21doa
++ mktemp
+ local LAST_ERR=/tmp/tmp.yLGfN1XqAf
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BgDKe21doa
+ cat /tmp/tmp.yLGfN1XqAf
+ rm /tmp/tmp.BgDKe21doa /tmp/tmp.yLGfN1XqAf
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ wait_restore backup-gcp-cs-sharded some-name ready 0 3000
+ local backup_name=backup-gcp-cs-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-gcp-cs-sharded to reach ready state.................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AMn0kA29iY
++ mktemp
+ local LAST_ERR=/tmp/tmp.HASrlcs9yr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AMn0kA29iY
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-16455"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-backup-mongodb:2.7.0","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
percona.com/resync-pbm: "true"
creationTimestamp: "2024-12-02T22:06:23Z"
generation: 2
name: some-name
namespace: demand-backup-physical-sharded-16455
resourceVersion: "45595"
uid: b160147e-1cf6-4530-ac52-8a83191c65d3
spec:
backup:
enabled: true
image: perconalab/percona-backup-mongodb:2.7.0
storages:
aws-s3:
s3:
bucket: operator-testing
credentialsSecret: aws-s3-secret
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
azure-blob:
azure:
container: operator-testing
credentialsSecret: azure-secret
prefix: psmdb-demand-backup-physical-sharded
type: azure
gcp-cs:
s3:
bucket: operator-testing
credentialsSecret: gcp-cs-secret
endpointUrl: https://storage.googleapis.com
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
minio:
s3:
bucket: operator-testing
credentialsSecret: minio-secret
endpointUrl: http://minio-service:9000/
insecureSkipTLSVerify: false
region: us-east-1
type: s3
tasks:
- compressionType: gzip
enabled: true
name: weekly
schedule: 0 0 * * 0
storageName: aws-s3
crVersion: 1.19.0
image: perconalab/percona-server-mongodb-operator:main-mongod7.0
imagePullPolicy: Always
replsets:
- affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
name: rs0
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
secrets:
users: some-users
sharding:
configsvrReplSet:
affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
enabled: true
mongos:
affinity:
antiAffinityTopologyKey: none
expose:
type: LoadBalancer
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
upgradeOptions:
apply: Never
status:
conditions:
- lastTransitionTime: "2024-12-02T22:25:33Z"
message: |-
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
reason: ErrorReconcile
status: "True"
type: error
- lastTransitionTime: "2024-12-02T22:25:55Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:27:25Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:27:25Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:27:50Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:27:50Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:28:05Z"
reason: MongosReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:33:07Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:33:46Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:33:46Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:33:59Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:33:59Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:34:19Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:34:19Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:34:32Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:34:32Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:34:53Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:34:53Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:35:06Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:35:06Z"
status: "True"
type: initializing
host: 146.148.39.140
mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
mongoVersion: 7.0.15-9
mongos:
ready: 0
size: 0
status: initializing
observedGeneration: 2
ready: 6
replsets:
cfg:
initialized: true
ready: 3
size: 3
status: ready
rs0:
added_as_shard: true
initialized: true
ready: 3
size: 3
status: ready
size: 6
state: initializing
+ cat /tmp/tmp.HASrlcs9yr
+ rm /tmp/tmp.AMn0kA29iY /tmp/tmp.HASrlcs9yr
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SUKIuJ4fuT
+++ mktemp
++ yq '.metadata.annotations."percona.com/resync-pbm"'
++ local LAST_ERR=/tmp/tmp.YJBhtYKOkI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.SUKIuJ4fuT
++ cat /tmp/tmp.YJBhtYKOkI
++ rm /tmp/tmp.SUKIuJ4fuT /tmp/tmp.YJBhtYKOkI
++ return 0
+ '[' true == null ']'
+ echo
+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kVQk6MRYlg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wSoGlsVpQk
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kVQk6MRYlg
++ cat /tmp/tmp.wSoGlsVpQk
++ rm /tmp/tmp.kVQk6MRYlg /tmp/tmp.wSoGlsVpQk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q0Afcl70CN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0j5SWq3Qtb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Q0Afcl70CN
++ cat /tmp/tmp.0j5SWq3Qtb
++ rm /tmp/tmp.Q0Afcl70CN /tmp/tmp.0j5SWq3Qtb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5ucjtAJrzC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EmQy7qnQOf
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5ucjtAJrzC
++ cat /tmp/tmp.EmQy7qnQOf
++ rm /tmp/tmp.5ucjtAJrzC /tmp/tmp.EmQy7qnQOf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mPI9UtGFRJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.54zcGXSbaS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mPI9UtGFRJ
++ cat /tmp/tmp.54zcGXSbaS
++ rm /tmp/tmp.mPI9UtGFRJ /tmp/tmp.54zcGXSbaS
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.a7tm8s6pRb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.t1HR47kGXv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.a7tm8s6pRb
++ cat /tmp/tmp.t1HR47kGXv
++ rm /tmp/tmp.a7tm8s6pRb /tmp/tmp.t1HR47kGXv
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kM7Dknbf9S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OX8O5yXKJN
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.kM7Dknbf9S
++ cat /tmp/tmp.OX8O5yXKJN
++ rm /tmp/tmp.kM7Dknbf9S /tmp/tmp.OX8O5yXKJN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7ZUsyxtRgZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bRy0YXs1wi
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.7ZUsyxtRgZ
++ cat /tmp/tmp.bRy0YXs1wi
++ rm /tmp/tmp.7ZUsyxtRgZ /tmp/tmp.bRy0YXs1wi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Frd5nFA54P
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LJae1ZfqiR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.Frd5nFA54P
++ cat /tmp/tmp.LJae1ZfqiR
++ rm /tmp/tmp.Frd5nFA54P /tmp/tmp.LJae1ZfqiR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.87I5sDRzCi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.f1LBAas4PR
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.87I5sDRzCi
++ cat /tmp/tmp.f1LBAas4PR
++ rm /tmp/tmp.87I5sDRzCi /tmp/tmp.f1LBAas4PR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1B2xZFODQT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XPb0OdEYsM
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.1B2xZFODQT
++ cat /tmp/tmp.XPb0OdEYsM
++ rm /tmp/tmp.1B2xZFODQT /tmp/tmp.XPb0OdEYsM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aAUN5zprJn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NNpPXpa6f0
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.aAUN5zprJn
++ cat /tmp/tmp.NNpPXpa6f0
++ rm /tmp/tmp.aAUN5zprJn /tmp/tmp.NNpPXpa6f0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WSO8TZN5ko
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KIRc1DbEjy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.WSO8TZN5ko
++ cat /tmp/tmp.KIRc1DbEjy
++ rm /tmp/tmp.WSO8TZN5ko /tmp/tmp.KIRc1DbEjy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d5c8aq0R9W
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HzFOmRolqS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.d5c8aq0R9W
++ cat /tmp/tmp.HzFOmRolqS
++ rm /tmp/tmp.d5c8aq0R9W /tmp/tmp.HzFOmRolqS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yTg07GW2IK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9Hv9CUmDnx
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.yTg07GW2IK
++ cat /tmp/tmp.9Hv9CUmDnx
++ rm /tmp/tmp.yTg07GW2IK /tmp/tmp.9Hv9CUmDnx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9dqHi2EsPV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sNvRpkERLJ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9dqHi2EsPV
++ cat /tmp/tmp.sNvRpkERLJ
++ rm /tmp/tmp.9dqHi2EsPV /tmp/tmp.sNvRpkERLJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QGIldPfl5Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kV4pKLYTHn
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QGIldPfl5Y
++ cat /tmp/tmp.kV4pKLYTHn
++ rm /tmp/tmp.QGIldPfl5Y /tmp/tmp.kV4pKLYTHn
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish.....................
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.StiszVjEIR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VqyHbbIWeu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.StiszVjEIR
++ cat /tmp/tmp.VqyHbbIWeu
++ rm /tmp/tmp.StiszVjEIR /tmp/tmp.VqyHbbIWeu
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.2m2EgivMRR
++ mktemp
+ local LAST_ERR=/tmp/tmp.N6rAiAHW31
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2m2EgivMRR
+ cat /tmp/tmp.N6rAiAHW31
+ rm /tmp/tmp.2m2EgivMRR /tmp/tmp.N6rAiAHW31
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.hqutgSWKJt/find-sharded
+ check_exported_mongos_service_endpoint 146.148.39.140
+ local host=146.148.39.140
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tQ5Zz0pAeY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wBn6XRsAJY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.tQ5Zz0pAeY
++ cat /tmp/tmp.wBn6XRsAJY
++ rm /tmp/tmp.tQ5Zz0pAeY /tmp/tmp.wBn6XRsAJY
++ return 0
+ '[' 146.148.39.140 '!=' 146.148.39.140 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mX7zJRlTwz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.N3sTLggLiz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.mX7zJRlTwz
++ cat /tmp/tmp.N3sTLggLiz
++ rm /tmp/tmp.mX7zJRlTwz /tmp/tmp.N3sTLggLiz
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.dzC5rR7nzh
++ mktemp
+ local LAST_ERR=/tmp/tmp.W2fuFPDNGq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.dzC5rR7nzh
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("5356f372-3d52-4ce4-a5cd-7e7f1d3238cc") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.W2fuFPDNGq
+ rm /tmp/tmp.dzC5rR7nzh /tmp/tmp.W2fuFPDNGq
+ return 0
+ echo 'check backup and restore -- azure-blob'
check backup and restore -- azure-blob
+ run_restore backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ /usr/bin/sed -e 's/backupName:/backupName: backup-azure-blob-sharded/'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's/name:/name: restore-backup-azure-blob-sharded/'
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2f9oKtrqfY
++ mktemp
+ local LAST_ERR=/tmp/tmp.jB5FAw3sdc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.2f9oKtrqfY
perconaservermongodbrestore.psmdb.percona.com/restore-backup-azure-blob-sharded created
+ cat /tmp/tmp.jB5FAw3sdc
+ rm /tmp/tmp.2f9oKtrqfY /tmp/tmp.jB5FAw3sdc
+ return 0
+ run_recovery_check backup-azure-blob-sharded _restore_sharded
+ local backup_name=backup-azure-blob-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-azure-blob-sharded some-name requested 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-azure-blob-sharded to reach requested state.....................................................................................................................
+ '[' 0 -eq 1 ']'
+ echo
+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
del(.metadata.ownerReferences[].apiVersion) |
del(.metadata.managedFields) |
del(.. | select(has("creationTimestamp")).creationTimestamp) |
del(.. | select(has("namespace")).namespace) |
del(.. | select(has("uid")).uid) |
del(.metadata.resourceVersion) |
del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
del(.metadata.selfLink) |
del(.metadata.annotations."cloud.google.com/neg") |
del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
del(.. | select(has("image")).image) |
del(.. | select(has("clusterIP")).clusterIP) |
del(.. | select(has("clusterIPs")).clusterIPs) |
del(.. | select(has("dataSource")).dataSource) |
del(.. | select(has("procMount")).procMount) |
del(.. | select(has("storageClassName")).storageClassName) |
del(.. | select(has("finalizers")).finalizers) |
del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
del(.. | select(has("volumeName")).volumeName) |
del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
del(.spec.volumeMode) |
del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
del(.. | select(has("nodePort")).nodePort) |
del(.status) |
(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-16455", "NAME_SPACE") |
del(.spec.volumeClaimTemplates[].apiVersion) |
del(.spec.volumeClaimTemplates[].kind) |
del(.spec.ipFamilies) |
del(.spec.ipFamilyPolicy) |
(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZHAIeTbjKd
++ mktemp
+ local LAST_ERR=/tmp/tmp.AKtLz3yes2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.ZHAIeTbjKd
+ cat /tmp/tmp.AKtLz3yes2
+ rm /tmp/tmp.ZHAIeTbjKd /tmp/tmp.AKtLz3yes2
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ wait_restore backup-azure-blob-sharded some-name ready 0 3000
+ local backup_name=backup-azure-blob-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-azure-blob-sharded to reach ready state.....................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5lojVMMCJO
++ mktemp
+ local LAST_ERR=/tmp/tmp.bfZYKDMJQD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.5lojVMMCJO
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-16455"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-backup-mongodb:2.7.0","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
percona.com/resync-pbm: "true"
creationTimestamp: "2024-12-02T22:06:23Z"
generation: 2
name: some-name
namespace: demand-backup-physical-sharded-16455
resourceVersion: "53913"
uid: b160147e-1cf6-4530-ac52-8a83191c65d3
spec:
backup:
enabled: true
image: perconalab/percona-backup-mongodb:2.7.0
storages:
aws-s3:
s3:
bucket: operator-testing
credentialsSecret: aws-s3-secret
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
azure-blob:
azure:
container: operator-testing
credentialsSecret: azure-secret
prefix: psmdb-demand-backup-physical-sharded
type: azure
gcp-cs:
s3:
bucket: operator-testing
credentialsSecret: gcp-cs-secret
endpointUrl: https://storage.googleapis.com
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
minio:
s3:
bucket: operator-testing
credentialsSecret: minio-secret
endpointUrl: http://minio-service:9000/
insecureSkipTLSVerify: false
region: us-east-1
type: s3
tasks:
- compressionType: gzip
enabled: true
name: weekly
schedule: 0 0 * * 0
storageName: aws-s3
crVersion: 1.19.0
image: perconalab/percona-server-mongodb-operator:main-mongod7.0
imagePullPolicy: Always
replsets:
- affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
name: rs0
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
secrets:
users: some-users
sharding:
configsvrReplSet:
affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
enabled: true
mongos:
affinity:
antiAffinityTopologyKey: none
expose:
type: LoadBalancer
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
upgradeOptions:
apply: Never
status:
conditions:
- lastTransitionTime: "2024-12-02T22:34:32Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:34:32Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:34:53Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:34:53Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:35:06Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:35:06Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:41:02Z"
message: |-
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
reason: ErrorReconcile
status: "True"
type: error
- lastTransitionTime: "2024-12-02T22:41:23Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:42:55Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:42:55Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:43:18Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:43:18Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:43:20Z"
reason: MongosReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:46:28Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:47:01Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:47:01Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:47:41Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:47:41Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:48:13Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:48:13Z"
status: "True"
type: initializing
host: 146.148.39.140
mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
mongoVersion: 7.0.15-9
mongos:
ready: 0
size: 0
status: initializing
observedGeneration: 2
ready: 6
replsets:
cfg:
initialized: true
ready: 3
size: 3
status: ready
rs0:
added_as_shard: true
initialized: true
ready: 3
size: 3
status: ready
size: 6
state: initializing
+ cat /tmp/tmp.bfZYKDMJQD
+ rm /tmp/tmp.5lojVMMCJO /tmp/tmp.bfZYKDMJQD
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9zfrb4Bhfy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6Vdn8Rc8rA
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9zfrb4Bhfy
++ cat /tmp/tmp.6Vdn8Rc8rA
++ rm /tmp/tmp.9zfrb4Bhfy /tmp/tmp.6Vdn8Rc8rA
++ return 0
+ '[' true == null ']'
+ echo
+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5YgFgXUO5O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bA9ZXr84r2
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.5YgFgXUO5O
++ cat /tmp/tmp.bA9ZXr84r2
++ rm /tmp/tmp.5YgFgXUO5O /tmp/tmp.bA9ZXr84r2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HwW20whwQa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RcQ44f4iiW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HwW20whwQa
++ cat /tmp/tmp.RcQ44f4iiW
++ rm /tmp/tmp.HwW20whwQa /tmp/tmp.RcQ44f4iiW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8Z6ez5ulTd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IMUHZ4HwCY
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8Z6ez5ulTd
++ cat /tmp/tmp.IMUHZ4HwCY
++ rm /tmp/tmp.8Z6ez5ulTd /tmp/tmp.IMUHZ4HwCY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nWSWKMC71c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xFtvybch10
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nWSWKMC71c
++ cat /tmp/tmp.xFtvybch10
++ rm /tmp/tmp.nWSWKMC71c /tmp/tmp.xFtvybch10
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8JdrF6uAcX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p1YkqzHNMW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8JdrF6uAcX
++ cat /tmp/tmp.p1YkqzHNMW
++ rm /tmp/tmp.8JdrF6uAcX /tmp/tmp.p1YkqzHNMW
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2zyxBsreNd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ocEPPD7gOw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2zyxBsreNd
++ cat /tmp/tmp.ocEPPD7gOw
++ rm /tmp/tmp.2zyxBsreNd /tmp/tmp.ocEPPD7gOw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wfaE6idflS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fg6EHVFku8
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wfaE6idflS
++ cat /tmp/tmp.fg6EHVFku8
++ rm /tmp/tmp.wfaE6idflS /tmp/tmp.fg6EHVFku8
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ATlqQv4hsa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TEQFSDxgSq
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.ATlqQv4hsa
++ cat /tmp/tmp.TEQFSDxgSq
++ rm /tmp/tmp.ATlqQv4hsa /tmp/tmp.TEQFSDxgSq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wo7ipTKfum
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jDXzVyCY1u
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wo7ipTKfum
++ cat /tmp/tmp.jDXzVyCY1u
++ rm /tmp/tmp.wo7ipTKfum /tmp/tmp.jDXzVyCY1u
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QzxFFkMSfs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dpe4kEK7Xl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QzxFFkMSfs
++ cat /tmp/tmp.dpe4kEK7Xl
++ rm /tmp/tmp.QzxFFkMSfs /tmp/tmp.dpe4kEK7Xl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.QcU9iOuOr9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U6OZynItqg
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.QcU9iOuOr9
++ cat /tmp/tmp.U6OZynItqg
++ rm /tmp/tmp.QcU9iOuOr9 /tmp/tmp.U6OZynItqg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.51SJuzHqEH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F5bKOrfJ28
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.51SJuzHqEH
++ cat /tmp/tmp.F5bKOrfJ28
++ rm /tmp/tmp.51SJuzHqEH /tmp/tmp.F5bKOrfJ28
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J2d1qJWepA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dTYP1kNbRZ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.J2d1qJWepA
++ cat /tmp/tmp.dTYP1kNbRZ
++ rm /tmp/tmp.J2d1qJWepA /tmp/tmp.dTYP1kNbRZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FnFlmccEyh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.5XJGGBOsMC
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.FnFlmccEyh
++ cat /tmp/tmp.5XJGGBOsMC
++ rm /tmp/tmp.FnFlmccEyh /tmp/tmp.5XJGGBOsMC
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nwSnVcvwjA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.g3h50KX25b
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.nwSnVcvwjA
++ cat /tmp/tmp.g3h50KX25b
++ rm /tmp/tmp.nwSnVcvwjA /tmp/tmp.g3h50KX25b
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2ToI4Uh7en
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XeQ1ewSL7R
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.2ToI4Uh7en
++ cat /tmp/tmp.XeQ1ewSL7R
++ rm /tmp/tmp.2ToI4Uh7en /tmp/tmp.XeQ1ewSL7R
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CHssPBZmKx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.akCpPxtLsd
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.CHssPBZmKx
++ cat /tmp/tmp.akCpPxtLsd
++ rm /tmp/tmp.CHssPBZmKx /tmp/tmp.akCpPxtLsd
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish..........
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cQPGXrdoVx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1lXUOsP7cl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cQPGXrdoVx
++ cat /tmp/tmp.1lXUOsP7cl
++ rm /tmp/tmp.cQPGXrdoVx /tmp/tmp.1lXUOsP7cl
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.GR40JPyUWU
++ mktemp
+ local LAST_ERR=/tmp/tmp.JmN47IXuqi
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.GR40JPyUWU
+ cat /tmp/tmp.JmN47IXuqi
+ rm /tmp/tmp.GR40JPyUWU /tmp/tmp.JmN47IXuqi
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.hqutgSWKJt/find-sharded
+ check_exported_mongos_service_endpoint 146.148.39.140
+ local host=146.148.39.140
++ kubectl_bin get psmdb some-name '-o=jsonpath={.status.host}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cD8KeRwX13
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TY7RUWnvuy
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name '-o=jsonpath={.status.host}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.cD8KeRwX13
++ cat /tmp/tmp.TY7RUWnvuy
++ rm /tmp/tmp.cD8KeRwX13 /tmp/tmp.TY7RUWnvuy
++ return 0
+ '[' 146.148.39.140 '!=' 146.148.39.140 ']'
+ echo 'drop collection'
drop collection
+ run_mongos 'use myApp\n db.test.drop()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local 'command=use myApp\n db.test.drop()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ local suffix=.svc.cluster.local
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.axphr8Rooi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QheLs9kHhD
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.axphr8Rooi
++ cat /tmp/tmp.QheLs9kHhD
++ rm /tmp/tmp.axphr8Rooi /tmp/tmp.QheLs9kHhD
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.TZb8eg4r4k
++ mktemp
+ local LAST_ERR=/tmp/tmp.l1JdQTZrKk
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.drop()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.TZb8eg4r4k
Percona Server for MongoDB shell version v4.4.29-28
connecting to: mongodb://some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local:27017/admin?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("b4f47579-c5d5-4800-8397-490f8f256b3f") }
Percona Server for MongoDB server version: v7.0.15-9
WARNING: shell and server versions do not match
switched to db myApp
true
bye
+ cat /tmp/tmp.l1JdQTZrKk
+ rm /tmp/tmp.TZb8eg4r4k /tmp/tmp.l1JdQTZrKk
+ return 0
+ echo 'check backup and restore -- minio'
check backup and restore -- minio
++ get_backup_dest backup-minio-sharded
++ local backup_name=backup-minio-sharded
++ kubectl_bin get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JxGtAWo6NZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cUclSGijtt
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ sed -e 's/.json$//'
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb-backup backup-minio-sharded -o 'jsonpath={.status.destination}'
++ sed 's|s3://||'
++ sed 's|azure://||'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.JxGtAWo6NZ
++ cat /tmp/tmp.cUclSGijtt
++ rm /tmp/tmp.JxGtAWo6NZ /tmp/tmp.cUclSGijtt
++ return 0
+ backup_dest_minio=operator-testing/2024-12-02T22:11:58Z
+ run_restore backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ /usr/bin/sed -e 's/name:/name: restore-backup-minio-sharded/'
+ /usr/bin/sed -e 's/backupName:/backupName: backup-minio-sharded/'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/conf/restore.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.sxmsWWQNHL
++ mktemp
+ local LAST_ERR=/tmp/tmp.CzyCFaNCSu
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.sxmsWWQNHL
perconaservermongodbrestore.psmdb.percona.com/restore-backup-minio-sharded created
+ cat /tmp/tmp.CzyCFaNCSu
+ rm /tmp/tmp.sxmsWWQNHL /tmp/tmp.CzyCFaNCSu
+ return 0
+ run_recovery_check backup-minio-sharded _restore_sharded
+ local backup_name=backup-minio-sharded
+ local compare_suffix=_restore_sharded
+ wait_restore backup-minio-sharded some-name requested 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=requested
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-minio-sharded to reach requested state.............................................................
+ '[' 0 -eq 1 ']'
+ echo
+ compare_kubectl statefulset/some-name-rs0 _restore_sharded
+ local resource=statefulset/some-name-rs0
+ local postfix=_restore_sharded
+ local skip_generation_check=
+ local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
+ local new_result=/tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-rs0
+ yq eval '
del(.metadata.ownerReferences[].apiVersion) |
del(.metadata.managedFields) |
del(.. | select(has("creationTimestamp")).creationTimestamp) |
del(.. | select(has("namespace")).namespace) |
del(.. | select(has("uid")).uid) |
del(.metadata.resourceVersion) |
del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
del(.metadata.selfLink) |
del(.metadata.annotations."cloud.google.com/neg") |
del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
del(.. | select(has("image")).image) |
del(.. | select(has("clusterIP")).clusterIP) |
del(.. | select(has("clusterIPs")).clusterIPs) |
del(.. | select(has("dataSource")).dataSource) |
del(.. | select(has("procMount")).procMount) |
del(.. | select(has("storageClassName")).storageClassName) |
del(.. | select(has("finalizers")).finalizers) |
del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
del(.. | select(has("volumeName")).volumeName) |
del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
del(.spec.volumeMode) |
del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
del(.. | select(has("nodePort")).nodePort) |
del(.status) |
(.. | select(tag == "!!str")) |= sub("demand-backup-physical-sharded-16455", "NAME_SPACE") |
del(.spec.volumeClaimTemplates[].apiVersion) |
del(.spec.volumeClaimTemplates[].kind) |
del(.spec.ipFamilies) |
del(.spec.ipFamilyPolicy) |
(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
(.. | select(. == "batch/v1beta1")) = "batch/v1" ' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kfnzaytqO7
++ mktemp
+ local LAST_ERR=/tmp/tmp.MqaVEHkPAn
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-rs0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.kfnzaytqO7
+ cat /tmp/tmp.MqaVEHkPAn
+ rm /tmp/tmp.kfnzaytqO7 /tmp/tmp.MqaVEHkPAn
+ return 0
+ yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ version_gt 1.22
++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml == */cronjob* ]]
+ '[' -n '' ']'
+ diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml /tmp/tmp.hqutgSWKJt/statefulset_some-name-rs0.yml
+ wait_restore backup-minio-sharded some-name ready 0 3000
+ local backup_name=backup-minio-sharded
+ local cluster_name=some-name
+ local target_state=ready
+ local wait_cluster_consistency=0
+ local wait_time=3000
+ set +o xtrace
waiting psmdb-restore/backup-minio-sharded to reach ready state...................................................................
+ '[' 0 -eq 1 ']'
+ kubectl_bin get psmdb some-name -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.hXVHUuGxNi
++ mktemp
+ local LAST_ERR=/tmp/tmp.EZufEY7OjO
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get psmdb some-name -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.hXVHUuGxNi
apiVersion: psmdb.percona.com/v1
kind: PerconaServerMongoDB
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"psmdb.percona.com/v1","kind":"PerconaServerMongoDB","metadata":{"annotations":{},"name":"some-name","namespace":"demand-backup-physical-sharded-16455"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-backup-mongodb:2.7.0","storages":{"aws-s3":{"s3":{"bucket":"operator-testing","credentialsSecret":"aws-s3-secret","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"azure-blob":{"azure":{"container":"operator-testing","credentialsSecret":"azure-secret","prefix":"psmdb-demand-backup-physical-sharded"},"type":"azure"},"gcp-cs":{"s3":{"bucket":"operator-testing","credentialsSecret":"gcp-cs-secret","endpointUrl":"https://storage.googleapis.com","insecureSkipTLSVerify":false,"prefix":"psmdb-demand-backup-physical-sharded","region":"us-east-1"},"type":"s3"},"minio":{"s3":{"bucket":"operator-testing","credentialsSecret":"minio-secret","endpointUrl":"http://minio-service:9000/","insecureSkipTLSVerify":false,"region":"us-east-1"},"type":"s3"}},"tasks":[{"compressionType":"gzip","enabled":true,"name":"weekly","schedule":"0 0 * * 0","storageName":"aws-s3"}]},"image":"perconalab/percona-server-mongodb-operator:main-mongod7.0","imagePullPolicy":"Always","replsets":[{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"name":"rs0","resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}}],"secrets":{"users":"some-users"},"sharding":{"configsvrReplSet":{"affinity":{"antiAffinityTopologyKey":"none"},"configuration":"operationProfiling:\n mode: slowOp\n slowOpThresholdMs: 100\nsecurity:\n enableEncryption: true\n redactClientLogData: false\nsetParameter:\n ttlMonitorSleepSecs: 60\n wiredTigerConcurrentReadTransactions: 128\n wiredTigerConcurrentWriteTransactions: 128\nstorage:\n engine: wiredTiger\n wiredTiger:\n collectionConfig:\n blockCompressor: snappy\n engineConfig:\n directoryForIndexes: false\n journalCompressor: snappy\n indexConfig:\n prefixCompression: true\n","expose":{"enabled":false,"type":"ClusterIP"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3,"volumeSpec":{"persistentVolumeClaim":{"resources":{"requests":{"storage":"3Gi"}}}}},"enabled":true,"mongos":{"affinity":{"antiAffinityTopologyKey":"none"},"expose":{"type":"LoadBalancer"},"resources":{"limits":{"cpu":"500m","memory":"1G"},"requests":{"cpu":"100m","memory":"0.1G"}},"size":3}},"upgradeOptions":{"apply":"Never"}}}
percona.com/resync-pbm: "true"
creationTimestamp: "2024-12-02T22:06:23Z"
generation: 2
name: some-name
namespace: demand-backup-physical-sharded-16455
resourceVersion: "59850"
uid: b160147e-1cf6-4530-ac52-8a83191c65d3
spec:
backup:
enabled: true
image: perconalab/percona-backup-mongodb:2.7.0
storages:
aws-s3:
s3:
bucket: operator-testing
credentialsSecret: aws-s3-secret
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
azure-blob:
azure:
container: operator-testing
credentialsSecret: azure-secret
prefix: psmdb-demand-backup-physical-sharded
type: azure
gcp-cs:
s3:
bucket: operator-testing
credentialsSecret: gcp-cs-secret
endpointUrl: https://storage.googleapis.com
insecureSkipTLSVerify: false
prefix: psmdb-demand-backup-physical-sharded
region: us-east-1
type: s3
minio:
s3:
bucket: operator-testing
credentialsSecret: minio-secret
endpointUrl: http://minio-service:9000/
insecureSkipTLSVerify: false
region: us-east-1
type: s3
tasks:
- compressionType: gzip
enabled: true
name: weekly
schedule: 0 0 * * 0
storageName: aws-s3
crVersion: 1.19.0
image: perconalab/percona-server-mongodb-operator:main-mongod7.0
imagePullPolicy: Always
replsets:
- affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
name: rs0
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
secrets:
users: some-users
sharding:
configsvrReplSet:
affinity:
antiAffinityTopologyKey: none
configuration: |
operationProfiling:
mode: slowOp
slowOpThresholdMs: 100
security:
enableEncryption: true
redactClientLogData: false
setParameter:
ttlMonitorSleepSecs: 60
wiredTigerConcurrentReadTransactions: 128
wiredTigerConcurrentWriteTransactions: 128
storage:
engine: wiredTiger
wiredTiger:
collectionConfig:
blockCompressor: snappy
engineConfig:
directoryForIndexes: false
journalCompressor: snappy
indexConfig:
prefixCompression: true
expose:
enabled: false
type: ClusterIP
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
volumeSpec:
persistentVolumeClaim:
resources:
requests:
storage: 3Gi
enabled: true
mongos:
affinity:
antiAffinityTopologyKey: none
expose:
type: LoadBalancer
resources:
limits:
cpu: 500m
memory: 1G
requests:
cpu: 100m
memory: 0.1G
size: 3
upgradeOptions:
apply: Never
status:
conditions:
- lastTransitionTime: "2024-12-02T22:54:10Z"
message: |-
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-cfg-0.some-name-cfg.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
handle ReplicaSetNoPrimary: get standalone mongo client: ping mongo: server selection error: server selection timeout, current topology: { Type: Single, Servers: [{ Addr: some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local:27017, Type: Unknown, Last error: dial tcp: lookup some-name-rs0-0.some-name-rs0.demand-backup-physical-sharded-16455.svc.cluster.local on 10.214.208.10:53: no such host }, ] }
reason: ErrorReconcile
status: "True"
type: error
- lastTransitionTime: "2024-12-02T22:54:31Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:56:02Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:56:02Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:56:31Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:56:31Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:56:33Z"
reason: MongosReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:58:14Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:58:47Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:58:47Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:59:00Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:59:00Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:59:26Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:59:26Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:59:32Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:59:32Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T22:59:54Z"
message: 'rs0: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T22:59:54Z"
status: "True"
type: initializing
- lastTransitionTime: "2024-12-02T23:00:12Z"
message: 'cfg: ready'
reason: RSReady
status: "True"
type: ready
- lastTransitionTime: "2024-12-02T23:00:12Z"
status: "True"
type: initializing
host: 146.148.39.140
mongoImage: perconalab/percona-server-mongodb-operator:main-mongod7.0
mongoVersion: 7.0.15-9
mongos:
ready: 0
size: 0
status: initializing
observedGeneration: 2
ready: 6
replsets:
cfg:
initialized: true
ready: 3
size: 3
status: ready
rs0:
added_as_shard: true
initialized: true
ready: 3
size: 3
status: ready
size: 6
state: initializing
+ cat /tmp/tmp.EZufEY7OjO
+ rm /tmp/tmp.hXVHUuGxNi /tmp/tmp.EZufEY7OjO
+ return 0
++ kubectl_bin get psmdb some-name -o yaml
++ yq '.metadata.annotations."percona.com/resync-pbm"'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gp1Z1r4ujP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NjrR7Nhm4Y
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o yaml
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.gp1Z1r4ujP
++ cat /tmp/tmp.NjrR7Nhm4Y
++ rm /tmp/tmp.gp1Z1r4ujP /tmp/tmp.NjrR7Nhm4Y
++ return 0
+ '[' true == null ']'
+ echo
+ wait_cluster_consistency some-name 42
+ local cluster_name=some-name
+ local wait_time=42
+ retry=0
+ sleep 7
+ echo -n 'waiting for cluster readyness'
waiting for cluster readyness++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.l7CTKAvCdv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OstvRt720W
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.l7CTKAvCdv
++ cat /tmp/tmp.OstvRt720W
++ rm /tmp/tmp.l7CTKAvCdv /tmp/tmp.OstvRt720W
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 1 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I9kLC8uXws
+++ mktemp
++ local LAST_ERR=/tmp/tmp.azgmpOrVh1
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.I9kLC8uXws
++ cat /tmp/tmp.azgmpOrVh1
++ rm /tmp/tmp.I9kLC8uXws /tmp/tmp.azgmpOrVh1
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 2 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lIzqf1gZJ6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gdbOgR9y2m
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.lIzqf1gZJ6
++ cat /tmp/tmp.gdbOgR9y2m
++ rm /tmp/tmp.lIzqf1gZJ6 /tmp/tmp.gdbOgR9y2m
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 3 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PXljPydsZm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1S8EXdNeyS
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.PXljPydsZm
++ cat /tmp/tmp.1S8EXdNeyS
++ rm /tmp/tmp.PXljPydsZm /tmp/tmp.1S8EXdNeyS
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 4 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.C8MQSXeXtI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WfrlPoPg89
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.C8MQSXeXtI
++ cat /tmp/tmp.WfrlPoPg89
++ rm /tmp/tmp.C8MQSXeXtI /tmp/tmp.WfrlPoPg89
++ return 0
+ [[ error == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 5 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KhBJEGX121
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nz0sXb3Yov
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.KhBJEGX121
++ cat /tmp/tmp.Nz0sXb3Yov
++ rm /tmp/tmp.KhBJEGX121 /tmp/tmp.Nz0sXb3Yov
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 6 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UAG2lJcdjY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EJnsPEP8aQ
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.UAG2lJcdjY
++ cat /tmp/tmp.EJnsPEP8aQ
++ rm /tmp/tmp.UAG2lJcdjY /tmp/tmp.EJnsPEP8aQ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 7 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HLFCCKCxAy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Tj5icpGGfz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.HLFCCKCxAy
++ cat /tmp/tmp.Tj5icpGGfz
++ rm /tmp/tmp.HLFCCKCxAy /tmp/tmp.Tj5icpGGfz
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 8 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OXEXt9dgfg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HwRb72sfWa
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.OXEXt9dgfg
++ cat /tmp/tmp.HwRb72sfWa
++ rm /tmp/tmp.OXEXt9dgfg /tmp/tmp.HwRb72sfWa
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 9 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RsNrGDuUAh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vq6g76B1Hb
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RsNrGDuUAh
++ cat /tmp/tmp.Vq6g76B1Hb
++ rm /tmp/tmp.RsNrGDuUAh /tmp/tmp.Vq6g76B1Hb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 10 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8OXTtZmSLd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BZk1LVKLrw
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.8OXTtZmSLd
++ cat /tmp/tmp.BZk1LVKLrw
++ rm /tmp/tmp.8OXTtZmSLd /tmp/tmp.BZk1LVKLrw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 11 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.X651AfhKP4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qnU9R4CucT
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.X651AfhKP4
++ cat /tmp/tmp.qnU9R4CucT
++ rm /tmp/tmp.X651AfhKP4 /tmp/tmp.qnU9R4CucT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 12 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wABrTQ90ec
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1cnBib7g4j
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.wABrTQ90ec
++ cat /tmp/tmp.1cnBib7g4j
++ rm /tmp/tmp.wABrTQ90ec /tmp/tmp.1cnBib7g4j
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 13 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.11INVegQEu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mf2bh5HD5o
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.11INVegQEu
++ cat /tmp/tmp.mf2bh5HD5o
++ rm /tmp/tmp.11INVegQEu /tmp/tmp.mf2bh5HD5o
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 14 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9PVCA4alrQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2gXc4vTxwI
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.9PVCA4alrQ
++ cat /tmp/tmp.2gXc4vTxwI
++ rm /tmp/tmp.9PVCA4alrQ /tmp/tmp.2gXc4vTxwI
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 15 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RdCW3sFVNv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2nE1lpRbGV
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.RdCW3sFVNv
++ cat /tmp/tmp.2nE1lpRbGV
++ rm /tmp/tmp.RdCW3sFVNv /tmp/tmp.2nE1lpRbGV
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ let retry+=1
+ '[' 16 -ge 42 ']'
+ echo -n .
.+ sleep 10
++ kubectl_bin get psmdb some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qXztB3Kh2E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WKxl8YynTz
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get psmdb some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.qXztB3Kh2E
++ cat /tmp/tmp.WKxl8YynTz
++ rm /tmp/tmp.qXztB3Kh2E /tmp/tmp.WKxl8YynTz
++ return 0
+ [[ ready == \r\e\a\d\y ]]
+ echo
+ wait_for_pbm_operations some-name
+ local cluster=some-name
+ set +o xtrace
waiting for PBM operation to finish
+ compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 -sharded
+ local command=find
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local postfix=-sharded
+ local suffix=
+ local database=myApp
+ local collection=test
+ run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455 mongodb ''
+ local 'command=use myApp\n db.test.find()'
+ local uri=myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455
+ local driver=mongodb
+ egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:'
+ local suffix=.svc.cluster.local
+ /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/'
++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NbaxH9ooYt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uqkgtFazOv
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.NbaxH9ooYt
++ cat /tmp/tmp.uqkgtFazOv
++ rm /tmp/tmp.NbaxH9ooYt /tmp/tmp.uqkgtFazOv
++ return 0
+ local client_container=psmdb-client-85c494dc46-gpsct
+ local mongo_flag=
+ kubectl_bin exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
++ mktemp
+ local LAST_OUT=/tmp/tmp.9EN38PQUH9
++ mktemp
+ local LAST_ERR=/tmp/tmp.T1L2jA4rMt
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl exec psmdb-client-85c494dc46-gpsct -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-physical-sharded-16455.svc.cluster.local/admin '
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9EN38PQUH9
+ cat /tmp/tmp.T1L2jA4rMt
+ rm /tmp/tmp.9EN38PQUH9 /tmp/tmp.T1L2jA4rMt
+ return 0
+ diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/e2e-tests/demand-backup-physical-sharded/compare/find-sharded.json /tmp/tmp.hqutgSWKJt/find-sharded
+ destroy demand-backup-physical-sharded-16455
+ local namespace=demand-backup-physical-sharded-16455
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace
-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------
+ '[' true == false ']'
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace
-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.JrNAUzpFEp
++ mktemp
+ local LAST_ERR=/tmp/tmp.xSE3Duok5P
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JrNAUzpFEp
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.xSE3Duok5P
+ rm /tmp/tmp.JrNAUzpFEp /tmp/tmp.xSE3Duok5P
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/crd.yaml
++ grep -v '\-\-\-'
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-16455 backup-aws-s3-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-16455 backup-azure-blob-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob-sharded patched
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-16455 backup-gcp-cs-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
Error from server (NotFound): perconaservermongodbbackups.psmdb.percona.com "backup-gcp-cs-sharded" not found
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n demand-backup-physical-sharded-16455 backup-minio-sharded --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaservermongodbbackup.psmdb.percona.com/backup-minio-sharded patched
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.DsrBLYTHOq
++ mktemp
+ local LAST_ERR=/tmp/tmp.jxNJKEBbhw
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.DsrBLYTHOq
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com condition met
+ cat /tmp/tmp.jxNJKEBbhw
+ rm /tmp/tmp.DsrBLYTHOq /tmp/tmp.jxNJKEBbhw
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.fdrgHV6vK7
++ mktemp
+ local LAST_ERR=/tmp/tmp.tmGoMRe9L2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fdrgHV6vK7
+ cat /tmp/tmp.tmGoMRe9L2
+ rm /tmp/tmp.fdrgHV6vK7 /tmp/tmp.tmGoMRe9L2
+ return 0
+ for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')'
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbs"
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbs"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.CiXEvM3ngM
++ mktemp
+ local LAST_ERR=/tmp/tmp.pxOY8uQ1Hv
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.CiXEvM3ngM
+ cat /tmp/tmp.pxOY8uQ1Hv
+ rm /tmp/tmp.CiXEvM3ngM /tmp/tmp.pxOY8uQ1Hv
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.Pf5q1yYBEV
++ mktemp
+ local LAST_ERR=/tmp/tmp.79B9ZIfGu9
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1746/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Pf5q1yYBEV
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.79B9ZIfGu9
+ rm /tmp/tmp.Pf5q1yYBEV /tmp/tmp.79B9ZIfGu9
+ return 0
+ destroy_cert_manager
+ kubectl_bin delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PuS0UNabqs
++ mktemp
+ local LAST_ERR=/tmp/tmp.4qtyhgFV8F
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PuS0UNabqs
+ cat /tmp/tmp.4qtyhgFV8F
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PuS0UNabqs
+ cat /tmp/tmp.4qtyhgFV8F
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 4
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 -a -n 1 ']'
+ cat /tmp/tmp.PuS0UNabqs
+ cat /tmp/tmp.4qtyhgFV8F
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ sleep 8
+ cat /tmp/tmp.PuS0UNabqs
+ cat /tmp/tmp.4qtyhgFV8F
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": namespaces "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificaterequests.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "certificates.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "challenges.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "clusterissuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "issuers.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": customresourcedefinitions.apiextensions.k8s.io "orders.acme.cert-manager.io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": serviceaccounts "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-cluster-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-view" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-edit" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterroles.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-issuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-clusterissuers" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificates" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-orders" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-challenges" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-ingress-shim" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-approve:cert-manager-io" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-controller-certificatesigningrequests" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": clusterrolebindings.rbac.authorization.k8s.io "cert-manager-webhook:subjectaccessreviews" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": roles.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cainjector:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager:leaderelection" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-cert-manager-tokenrequest" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": rolebindings.rbac.authorization.k8s.io "cert-manager-webhook:dynamic-serving" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": services "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-cainjector" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": deployments.apps "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": mutatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
Error from server (NotFound): error when deleting "https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml": validatingwebhookconfigurations.admissionregistration.k8s.io "cert-manager-webhook" not found
+ rm /tmp/tmp.PuS0UNabqs /tmp/tmp.4qtyhgFV8F
+ return 1
+ true
+ '[' -n '' ']'
+ '[' -n psmdb-operator ']'
+ rm -rf /tmp/tmp.hqutgSWKJt
+ kubectl_bin delete --grace-period=0 --force=true namespace psmdb-operator
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-physical-sharded-16455
++ mktemp
+ local LAST_OUT=/tmp/tmp.6volFuSGXH
++ mktemp
+ local LAST_ERR=/tmp/tmp.di9TZvcX3K
+ local exit_status=0
+ local timeout=4
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-physical-sharded-16455
+ local LAST_OUT=/tmp/tmp.yFRA59PawY
++ mktemp
+ local LAST_ERR=/tmp/tmp.9rZnlukZbc
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace psmdb-operator