++ echo 'Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/logs/demand-backup-sharded.log' Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/logs/demand-backup-sharded.log ++ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP_AZURE= ++ oc get projects ++ kubectl get nodes ++ grep '^minikube' +++ grep '\-eks\-' +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep gke WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 ++ '[' v1.30.12-gke.1033000 ']' ++ GKE=1 +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.33) and server (1.30) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.30 + set_debug + [[ 1 == 1 ]] + set -o xtrace + custom_port=27019 + create_infra demand-backup-sharded-24520 + local ns=demand-backup-sharded-24520 + delete_crd + desc 'get and delete old CRDs and RBAC' + set +o xtrace ----------------------------------------------------------------------------------- get and delete old CRDs and RBAC ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml --ignore-not-found --wait=false ++ mktemp + local LAST_OUT=/tmp/tmp.XFzWNSQdlX ++ mktemp + local LAST_ERR=/tmp/tmp.vSAzUWilL3 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml --ignore-not-found --wait=false + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.XFzWNSQdlX customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted + cat /tmp/tmp.vSAzUWilL3 + rm /tmp/tmp.XFzWNSQdlX /tmp/tmp.vSAzUWilL3 + return 0 ++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml ++ grep -v '\-\-\-' + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.OqmoYB0AaU ++ mktemp + local LAST_ERR=/tmp/tmp.5Y5ZXR6YBN + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OqmoYB0AaU + cat /tmp/tmp.5Y5ZXR6YBN + rm /tmp/tmp.OqmoYB0AaU /tmp/tmp.5Y5ZXR6YBN + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + grep -v NAMESPACE + kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbrestores" + kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbrestores" + : + kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.K8qHrOQC5o ++ mktemp + local LAST_ERR=/tmp/tmp.QbaosXAHUC + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.K8qHrOQC5o + cat /tmp/tmp.QbaosXAHUC + rm /tmp/tmp.K8qHrOQC5o /tmp/tmp.QbaosXAHUC + return 0 + for crd_name in '$(yq eval '\''.metadata.name'\'' "${src_dir}/deploy/crd.yaml" | grep -v '\''\-\-\-'\'')' + kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' error: the server doesn't have a resource type "perconaservermongodbs" + kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: the server doesn't have a resource type "perconaservermongodbs" + : + kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.L1htHNcbXQ ++ mktemp + local LAST_ERR=/tmp/tmp.gwEWZc783A + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.L1htHNcbXQ + cat /tmp/tmp.gwEWZc783A + rm /tmp/tmp.L1htHNcbXQ /tmp/tmp.gwEWZc783A + return 0 + local rbac_yaml=rbac.yaml + '[' -n psmdb-operator ']' + rbac_yaml=cw-rbac.yaml + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.peo8WDd6w1 ++ mktemp + local LAST_ERR=/tmp/tmp.8kraFNI6Qg + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.peo8WDd6w1 clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted + cat /tmp/tmp.8kraFNI6Qg + rm /tmp/tmp.peo8WDd6w1 /tmp/tmp.8kraFNI6Qg + return 0 + check_crd_for_deletion PR-1904-fdb74d1e + local git_tag=PR-1904-fdb74d1e ++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-1904-fdb74d1e/deploy/crd.yaml ++ /usr/bin/sed ':a;N;$!ba;s/\n/ /g' ++ /usr/bin/sed s/---//g ++ yq eval .metadata.name + for crd_name in '$(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '\''.metadata.name'\'' | $sed '\''s/---//g'\'' | $sed '\'':a;N;$!ba;s/\n/ /g'\'')' ++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.aUstFtCR9D +++ mktemp ++ local LAST_ERR=/tmp/tmp.vi7YdVJI1U ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aUstFtCR9D ++ cat /tmp/tmp.vi7YdVJI1U Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 0 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aUstFtCR9D ++ cat /tmp/tmp.vi7YdVJI1U Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 4 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}' ++ exit_status=1 ++ set -e ++ '[' 1 '!=' 0 -a -n 1 ']' ++ cat /tmp/tmp.aUstFtCR9D ++ cat /tmp/tmp.vi7YdVJI1U Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ sleep 8 ++ cat /tmp/tmp.aUstFtCR9D ++ cat /tmp/tmp.vi7YdVJI1U Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found ++ rm /tmp/tmp.aUstFtCR9D /tmp/tmp.vi7YdVJI1U ++ return 1 + [[ '' == \T\e\r\m\i\n\a\t\i\n\g ]] + '[' -n psmdb-operator ']' + create_namespace psmdb-operator + local namespace=psmdb-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ sed s/NAMESPACE// ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace psmdb-operator --ignore-not-found ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.uzWauhME7i ++ mktemp + local LAST_OUT=/tmp/tmp.nZ2ckifxdJ + xargs kubectl delete ns ++ mktemp + local LAST_ERR=/tmp/tmp.Gda9kNrYaY + local exit_status=0 + local timeout=4 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.Ym4MAluFh8 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace psmdb-operator --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.uzWauhME7i + cat /tmp/tmp.Gda9kNrYaY + rm /tmp/tmp.uzWauhME7i /tmp/tmp.Gda9kNrYaY + return 0 namespace "demand-backup-sharded-10646" deleted namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.nZ2ckifxdJ namespace "psmdb-operator" deleted + cat /tmp/tmp.Ym4MAluFh8 + rm /tmp/tmp.nZ2ckifxdJ /tmp/tmp.Ym4MAluFh8 + return 0 + kubectl_bin wait --for=delete namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.HQ1HsNJewr ++ mktemp + local LAST_ERR=/tmp/tmp.jqkJtjxZx7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.HQ1HsNJewr + cat /tmp/tmp.jqkJtjxZx7 + rm /tmp/tmp.HQ1HsNJewr /tmp/tmp.jqkJtjxZx7 + return 0 + desc 'create namespace psmdb-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace psmdb-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.yQfj1wSNFw ++ mktemp + local LAST_ERR=/tmp/tmp.2KlGvObT06 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.yQfj1wSNFw namespace/psmdb-operator created + cat /tmp/tmp.2KlGvObT06 + rm /tmp/tmp.yQfj1wSNFw /tmp/tmp.2KlGvObT06 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.u091yS0Wi7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.61izf8lieZ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.u091yS0Wi7 ++ cat /tmp/tmp.61izf8lieZ ++ rm /tmp/tmp.u091yS0Wi7 /tmp/tmp.61izf8lieZ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7 --namespace=psmdb-operator ++ mktemp + local LAST_OUT=/tmp/tmp.rU55Kw8p3A ++ mktemp + local LAST_ERR=/tmp/tmp.UWoNIeLkxd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7 --namespace=psmdb-operator + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.rU55Kw8p3A Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7" modified. + cat /tmp/tmp.UWoNIeLkxd + rm /tmp/tmp.rU55Kw8p3A /tmp/tmp.UWoNIeLkxd + return 0 + deploy_operator + desc 'start PSMDB operator' + set +o xtrace ----------------------------------------------------------------------------------- start PSMDB operator ----------------------------------------------------------------------------------- + local cr_file + '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/crd.yaml ']' + cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2hnjLzFh3h ++ mktemp + local LAST_ERR=/tmp/tmp.ITYPoTQ1bL + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.2hnjLzFh3h customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied + cat /tmp/tmp.ITYPoTQ1bL + rm /tmp/tmp.2hnjLzFh3h /tmp/tmp.ITYPoTQ1bL + return 0 + '[' -n psmdb-operator ']' + apply_rbac cw-rbac + local operator_namespace=psmdb-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: psmdb-operator^' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-rbac.yaml + kubectl_bin apply -n psmdb-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.6yWQrN4TdF ++ mktemp + local LAST_ERR=/tmp/tmp.NCk2Myu7L0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n psmdb-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6yWQrN4TdF clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created serviceaccount/percona-server-mongodb-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created + cat /tmp/tmp.NCk2Myu7L0 + rm /tmp/tmp.6yWQrN4TdF /tmp/tmp.NCk2Myu7L0 + return 0 + yq eval ' (.spec.template.spec.containers[].image = "perconalab/percona-server-mongodb-operator:PR-1904-fdb74d1e") | ((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") | ((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/deploy/cw-operator.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Lo3al8ysE8 ++ mktemp + local LAST_ERR=/tmp/tmp.WKf9HuMBW7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Lo3al8ysE8 deployment.apps/percona-server-mongodb-operator created + cat /tmp/tmp.WKf9HuMBW7 + rm /tmp/tmp.Lo3al8ysE8 /tmp/tmp.WKf9HuMBW7 + return 0 + sleep 2 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.lSZWzcaWZz +++ mktemp ++ local LAST_ERR=/tmp/tmp.PZ2Xmpju7q ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.lSZWzcaWZz ++ cat /tmp/tmp.PZ2Xmpju7q ++ rm /tmp/tmp.lSZWzcaWZz /tmp/tmp.PZ2Xmpju7q ++ return 0 + wait_pod percona-server-mongodb-operator-c4f9b4485-tw7zb + local pod=percona-server-mongodb-operator-c4f9b4485-tw7zb + set +o xtrace waiting for pod/percona-server-mongodb-operator-c4f9b4485-tw7zb to be ready.OK + create_namespace demand-backup-sharded-24520 + local namespace=demand-backup-sharded-24520 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl api-resources ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-sharded-24520' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-sharded-24520 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-sharded-24520 --ignore-not-found + xargs kubectl delete ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|psmdb-operator|openshift|gke-mcs|^NAME' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.5IaXiyE1yZ + local LAST_OUT=/tmp/tmp.hgvfm8REYV ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.ccWiqCsB8s + local exit_status=0 + local timeout=4 + local LAST_ERR=/tmp/tmp.rHAtz2fOBr + local exit_status=0 + local timeout=4 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-sharded-24520 --ignore-not-found + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hgvfm8REYV + cat /tmp/tmp.rHAtz2fOBr + rm /tmp/tmp.hgvfm8REYV /tmp/tmp.rHAtz2fOBr + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.5IaXiyE1yZ + cat /tmp/tmp.ccWiqCsB8s + rm /tmp/tmp.5IaXiyE1yZ /tmp/tmp.ccWiqCsB8s + return 0 + kubectl_bin wait --for=delete namespace demand-backup-sharded-24520 ++ mktemp + local LAST_OUT=/tmp/tmp.z6XC9rRFCp ++ mktemp + local LAST_ERR=/tmp/tmp.hT2fXyVr2T + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=delete namespace demand-backup-sharded-24520 namespace "gke-managed-cim" deleted namespace "gke-managed-system" deleted namespace "gmp-public" deleted namespace "gmp-system" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.z6XC9rRFCp + cat /tmp/tmp.hT2fXyVr2T + rm /tmp/tmp.z6XC9rRFCp /tmp/tmp.hT2fXyVr2T + return 0 + desc 'create namespace demand-backup-sharded-24520' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-sharded-24520 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-sharded-24520 ++ mktemp + local LAST_OUT=/tmp/tmp.12pcIdJ5dc ++ mktemp + local LAST_ERR=/tmp/tmp.b9geRGxpcI + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace demand-backup-sharded-24520 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.12pcIdJ5dc namespace/demand-backup-sharded-24520 created + cat /tmp/tmp.b9geRGxpcI + rm /tmp/tmp.12pcIdJ5dc /tmp/tmp.b9geRGxpcI + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.dXA8aYYFjQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.JortSgBadX ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.dXA8aYYFjQ ++ cat /tmp/tmp.JortSgBadX ++ rm /tmp/tmp.dXA8aYYFjQ /tmp/tmp.JortSgBadX ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7 --namespace=demand-backup-sharded-24520 ++ mktemp + local LAST_OUT=/tmp/tmp.sxHpjiO9Pj ++ mktemp + local LAST_ERR=/tmp/tmp.LdSEXC7bt7 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7 --namespace=demand-backup-sharded-24520 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sxHpjiO9Pj Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-1904-fdb74d1e-9-cluster7" modified. + cat /tmp/tmp.LdSEXC7bt7 + rm /tmp/tmp.sxHpjiO9Pj /tmp/tmp.LdSEXC7bt7 + return 0 + deploy_minio + desc 'install Minio' + set +o xtrace ----------------------------------------------------------------------------------- install Minio ----------------------------------------------------------------------------------- + helm uninstall minio-service Error: uninstall: Release not loaded: minio-service: release: not found + : + helm repo remove minio "minio" has been removed from your repositories + helm repo add minio https://charts.min.io/ "minio" has been added to your repositories + retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio + local max=10 + local delay=60 + shift 2 + local n=1 + helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio NAME: minio-service LAST DEPLOYED: Mon May 12 13:00:28 2025 NAMESPACE: demand-backup-sharded-24520 STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: MinIO can be accessed via port 9000 on the following DNS name from within your cluster: minio-service.demand-backup-sharded-24520.cluster.local To access MinIO from localhost, run the below commands: 1. export POD_NAME=$(kubectl get pods --namespace demand-backup-sharded-24520 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") 2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-sharded-24520 Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-sharded-24520 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-sharded-24520 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 3. mc ls minio-service-local ++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yij0HvfRkX +++ mktemp ++ local LAST_ERR=/tmp/tmp.yJCXsfq6rq ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yij0HvfRkX ++ cat /tmp/tmp.yJCXsfq6rq ++ rm /tmp/tmp.yij0HvfRkX /tmp/tmp.yJCXsfq6rq ++ return 0 + MINIO_POD=minio-service-86dfccd949-kv9gt + wait_pod minio-service-86dfccd949-kv9gt + local pod=minio-service-86dfccd949-kv9gt + set +o xtrace waiting for pod/minio-service-86dfccd949-kv9gt to be ready.OK + '[' -n psmdb-operator ']' + kubectl_bin create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-24520.svc.cluster.local --tcp=9000 ++ mktemp + local LAST_OUT=/tmp/tmp.cd8SnKpvVl ++ mktemp + local LAST_ERR=/tmp/tmp.fHrAsMJV6t + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create svc -n psmdb-operator externalname minio-service --external-name=minio-service.demand-backup-sharded-24520.svc.cluster.local --tcp=9000 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cd8SnKpvVl service/minio-service created + cat /tmp/tmp.fHrAsMJV6t + rm /tmp/tmp.cd8SnKpvVl /tmp/tmp.fHrAsMJV6t + return 0 + create_minio_bucket operator-testing + local bucket=operator-testing + kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' ++ mktemp + local LAST_OUT=/tmp/tmp.mrrSgdo95m ++ mktemp + local LAST_ERR=/tmp/tmp.AmSoOLfN3P + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.mrrSgdo95m make_bucket: operator-testing pod "aws-cli" deleted + cat /tmp/tmp.AmSoOLfN3P If you don't see a command prompt, try pressing enter. warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: Internal error occurred: error attaching to container: container is in CONTAINER_EXITED state + rm /tmp/tmp.mrrSgdo95m /tmp/tmp.AmSoOLfN3P + return 0 + desc 'create first PSMDB cluster with custom port 27019' + set +o xtrace ----------------------------------------------------------------------------------- create first PSMDB cluster with custom port 27019 ----------------------------------------------------------------------------------- + cluster=some-name + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MPURpciSXU ++ mktemp + local LAST_ERR=/tmp/tmp.V3euZ9ZWQE + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.MPURpciSXU secret/some-users created deployment.apps/psmdb-client created + cat /tmp/tmp.V3euZ9ZWQE + rm /tmp/tmp.MPURpciSXU /tmp/tmp.V3euZ9ZWQE + return 0 + apply_s3_storage_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.AE8cgcZh4c ++ mktemp + local LAST_ERR=/tmp/tmp.FCzwhOVlAT + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AE8cgcZh4c secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.FCzwhOVlAT + rm /tmp/tmp.AE8cgcZh4c /tmp/tmp.FCzwhOVlAT + return 0 + version_gt 1.19 ++ echo '1.30 >= 1.19' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' 0 -ne 1 ']' + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/conf/container-rc.yaml + kubectl_bin apply -f - + /usr/bin/sed s/docker/runc/g ++ mktemp + local LAST_OUT=/tmp/tmp.s6pqKIT8ds ++ mktemp + local LAST_ERR=/tmp/tmp.2y3s22tY7V + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.s6pqKIT8ds runtimeclass.node.k8s.io/container-rc unchanged + cat /tmp/tmp.2y3s22tY7V + rm /tmp/tmp.s6pqKIT8ds /tmp/tmp.2y3s22tY7V + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/some-name-rs0.yml + yq eval '(.spec | select(.image == null)).image = "perconalab/percona-server-mongodb-operator:main-mongod7.0"' ++ mktemp + yq eval '(.spec | select(has("backup"))).backup.image = "perconalab/percona-server-mongodb-operator:main-backup"' + local LAST_OUT=/tmp/tmp.Bokwwc32Zc + yq eval '(.spec | select(has("initImage"))).initImage = "perconalab/percona-server-mongodb-operator:PR-1904-fdb74d1e"' + yq eval '(.spec | select(has("pmm"))).pmm.image = "perconalab/pmm-client:dev-latest"' + yq eval '.spec.upgradeOptions.apply="Never"' ++ mktemp + local LAST_ERR=/tmp/tmp.xvd3NR705d + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bokwwc32Zc perconaservermongodb.psmdb.percona.com/some-name created + cat /tmp/tmp.xvd3NR705d + rm /tmp/tmp.Bokwwc32Zc /tmp/tmp.xvd3NR705d + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- + wait_for_running some-name-rs0 3 + local name=some-name-rs0 + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=rs0 + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-rs0-0 + local pod=some-name-rs0-0 + set +o xtrace waiting for pod/some-name-rs0-0 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-rs0-1 + local pod=some-name-rs0-1 + set +o xtrace waiting for pod/some-name-rs0-1 to be ready......OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Fhbd57ou85 +++ mktemp ++ local LAST_ERR=/tmp/tmp.SMb19xO4wN ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.Fhbd57ou85 ++ cat /tmp/tmp.SMb19xO4wN ++ rm /tmp/tmp.Fhbd57ou85 /tmp/tmp.SMb19xO4wN ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-rs0-2 + local pod=some-name-rs0-2 + set +o xtrace waiting for pod/some-name-rs0-2 to be ready..........OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.R3aFGcygyK +++ mktemp ++ local LAST_ERR=/tmp/tmp.tjE89LYx3I ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="rs0")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.R3aFGcygyK ++ cat /tmp/tmp.tjE89LYx3I ++ rm /tmp/tmp.R3aFGcygyK /tmp/tmp.tjE89LYx3I ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness...................................... + wait_for_running some-name-cfg 3 false + local name=some-name-cfg + let last_pod=2 + local check_cluster_readyness=false + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=cfg + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-cfg-0 + local pod=some-name-cfg-0 + set +o xtrace waiting for pod/some-name-cfg-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-cfg-1 + local pod=some-name-cfg-1 + set +o xtrace waiting for pod/some-name-cfg-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qQhCqKKzhg +++ mktemp ++ local LAST_ERR=/tmp/tmp.ibTEn3qmnu ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.qQhCqKKzhg ++ cat /tmp/tmp.ibTEn3qmnu ++ rm /tmp/tmp.qQhCqKKzhg /tmp/tmp.ibTEn3qmnu ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-cfg-2 + local pod=some-name-cfg-2 + set +o xtrace waiting for pod/some-name-cfg-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MJ4W4yLfS5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.uxfrcLe7v1 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="cfg")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.MJ4W4yLfS5 ++ cat /tmp/tmp.uxfrcLe7v1 ++ rm /tmp/tmp.MJ4W4yLfS5 /tmp/tmp.uxfrcLe7v1 ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ false == \t\r\u\e ]] + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zxxZYBDJRu +++ mktemp ++ local LAST_ERR=/tmp/tmp.fpQd0i4ZtH ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.zxxZYBDJRu ++ cat /tmp/tmp.fpQd0i4ZtH ++ rm /tmp/tmp.zxxZYBDJRu /tmp/tmp.fpQd0i4ZtH ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GI5muGvg0x +++ mktemp ++ local LAST_ERR=/tmp/tmp.3bFJwro7OL ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.GI5muGvg0x ++ cat /tmp/tmp.3bFJwro7OL ++ rm /tmp/tmp.GI5muGvg0x /tmp/tmp.3bFJwro7OL ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness + sleep 10 + desc 'check if service and statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if service and statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/some-name-rs0 + local resource=statefulset/some-name-rs0 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs0.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs0 ++ mktemp + local LAST_OUT=/tmp/tmp.EDH7Ep3gjQ ++ mktemp + local LAST_ERR=/tmp/tmp.qfKK9ELnc0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs0 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.EDH7Ep3gjQ + cat /tmp/tmp.qfKK9ELnc0 + rm /tmp/tmp.EDH7Ep3gjQ /tmp/tmp.qfKK9ELnc0 + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs0.yml + version_gt 1.22 ++ bc -l ++ echo '1.30 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs0.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs0.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs0.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs0.yml + compare_kubectl statefulset/some-name-rs1 + local resource=statefulset/some-name-rs1 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs1.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs1 ++ mktemp + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.AQYS2ETnlH ++ mktemp + local LAST_ERR=/tmp/tmp.gcFvmeogWB + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs1 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.AQYS2ETnlH + cat /tmp/tmp.gcFvmeogWB + rm /tmp/tmp.AQYS2ETnlH /tmp/tmp.gcFvmeogWB + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs1.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs1.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs1.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs1.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs1.yml + compare_kubectl statefulset/some-name-rs2 + local resource=statefulset/some-name-rs2 + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs2.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-rs2 + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.wXoDAKStp3 ++ mktemp + local LAST_ERR=/tmp/tmp.EbOY10ssLd + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-rs2 + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wXoDAKStp3 + cat /tmp/tmp.EbOY10ssLd + rm /tmp/tmp.wXoDAKStp3 /tmp/tmp.EbOY10ssLd + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs2.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs2.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs2.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-rs2.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-rs2.yml + compare_kubectl statefulset/some-name-cfg + local resource=statefulset/some-name-cfg + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-cfg.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-cfg + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.hoKbsVqruJ ++ mktemp + local LAST_ERR=/tmp/tmp.t3Xno0EAuk + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-cfg + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hoKbsVqruJ + cat /tmp/tmp.t3Xno0EAuk + rm /tmp/tmp.hoKbsVqruJ /tmp/tmp.t3Xno0EAuk + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-cfg.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-cfg.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-cfg.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-cfg.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-cfg.yml + compare_kubectl statefulset/some-name-mongos '' + local resource=statefulset/some-name-mongos + local postfix= + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-mongos ++ mktemp + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - + local LAST_OUT=/tmp/tmp.U4trggIjfQ ++ mktemp + local LAST_ERR=/tmp/tmp.TXRe6slafA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.U4trggIjfQ + cat /tmp/tmp.TXRe6slafA + rm /tmp/tmp.U4trggIjfQ /tmp/tmp.TXRe6slafA + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.leqGmzIgf2 ++ mktemp + local LAST_ERR=/tmp/tmp.US0PjjEkVu + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/mongos-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.leqGmzIgf2 secret/some-name-mongos created + cat /tmp/tmp.US0PjjEkVu + rm /tmp/tmp.leqGmzIgf2 /tmp/tmp.US0PjjEkVu + return 0 + sleep 10 + wait_for_running some-name-mongos 3 + local name=some-name-mongos + let last_pod=2 + local check_cluster_readyness=true + set_debug + [[ 1 == 1 ]] + set -o xtrace + local rs_name=mongos + local cluster_name=some-name ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + [[ 0 -eq 2 ]] + wait_pod some-name-mongos-0 + local pod=some-name-mongos-0 + set +o xtrace waiting for pod/some-name-mongos-0 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 1 -eq 2 ]] + wait_pod some-name-mongos-1 + local pod=some-name-mongos-1 + set +o xtrace waiting for pod/some-name-mongos-1 to be ready.OK + for i in '$(seq 0 $last_pod)' + [[ 2 -eq 2 ]] ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PjcBi0c7CD +++ mktemp ++ local LAST_ERR=/tmp/tmp.T32OePYidI ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].arbiter.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.PjcBi0c7CD ++ cat /tmp/tmp.T32OePYidI ++ rm /tmp/tmp.PjcBi0c7CD /tmp/tmp.T32OePYidI ++ return 0 + [[ '' == \t\r\u\e ]] + wait_pod some-name-mongos-2 + local pod=some-name-mongos-2 + set +o xtrace waiting for pod/some-name-mongos-2 to be ready.OK ++ kubectl_bin get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZkU4I9SPCT +++ mktemp ++ local LAST_ERR=/tmp/tmp.p3gnXVldKx ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get psmdb some-name -o 'jsonpath={.spec.replsets[?(@.name=="mongos")].non_voting.enabled}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.ZkU4I9SPCT ++ cat /tmp/tmp.p3gnXVldKx ++ rm /tmp/tmp.ZkU4I9SPCT /tmp/tmp.p3gnXVldKx ++ return 0 + [[ '' == \t\r\u\e ]] + sleep 10 + [[ true == \t\r\u\e ]] + set +x Waiting for cluster readyness.......................................................................................... + compare_kubectl statefulset/some-name-mongos -secret + local resource=statefulset/some-name-mongos + local postfix=-secret + local skip_generation_check= + local expected_result=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml + local new_result=/tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + '[' -n '' -a -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret-oc.yml ']' + kubectl_bin get -o yaml statefulset/some-name-mongos + yq eval ' del(.metadata.ownerReferences[].apiVersion) | del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) | del(.metadata.selfLink) | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.status) | (.. | select(tag == "!!str")) |= sub("demand-backup-sharded-24520", "NAME_SPACE") | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | (.. | select(. == "extensions/v1beta1")) = "apps/v1" | (.. | select(. == "batch/v1beta1")) = "batch/v1" ' - ++ mktemp + local LAST_OUT=/tmp/tmp.RzoYn9aWLI ++ mktemp + local LAST_ERR=/tmp/tmp.djuJJXVeEp + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml statefulset/some-name-mongos + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.RzoYn9aWLI + cat /tmp/tmp.djuJJXVeEp + rm /tmp/tmp.RzoYn9aWLI /tmp/tmp.djuJJXVeEp + return 0 + yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + version_gt 1.22 ++ echo '1.30 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + yq -i eval 'del(.spec.internalTrafficPolicy)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + [[ /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml == */cronjob* ]] + '[' -n '' ']' + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/statefulset_some-name-mongos-secret.yml /tmp/tmp.fmiE2aOTZS/statefulset_some-name-mongos.yml + desc 'write data, read from all' + set +o xtrace ----------------------------------------------------------------------------------- write data, read from all ----------------------------------------------------------------------------------- + run_mongos 'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})' + local uri=userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.K7u2RfM2zl +++ mktemp ++ local LAST_ERR=/tmp/tmp.leyJ7UYXSQ ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.K7u2RfM2zl ++ cat /tmp/tmp.leyJ7UYXSQ ++ rm /tmp/tmp.K7u2RfM2zl /tmp/tmp.leyJ7UYXSQ ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.OvIypgNPfT ++ mktemp + local LAST_ERR=/tmp/tmp.rpcH3VrW3Y + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"},{db:"myApp1",role:"readWrite"},{db:"myApp2",role:"readWrite"}]})\n'\'' | mongo mongodb://userAdmin:userAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.OvIypgNPfT Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("0e1536cb-23b2-436b-8a4b-5f9c7fe89e7d") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match Successfully added user: { "user" : "myApp", "roles" : [ { "db" : "myApp", "role" : "readWrite" }, { "db" : "myApp1", "role" : "readWrite" }, { "db" : "myApp2", "role" : "readWrite" } ] } bye + cat /tmp/tmp.rpcH3VrW3Y + rm /tmp/tmp.OvIypgNPfT /tmp/tmp.rpcH3VrW3Y + return 0 + run_mongos 'sh.enableSharding("myApp","rs0")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=sh.enableSharding("myApp","rs0")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.cqaASjXn2K +++ mktemp ++ local LAST_ERR=/tmp/tmp.3TnXwFRHY3 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.cqaASjXn2K ++ cat /tmp/tmp.3TnXwFRHY3 ++ rm /tmp/tmp.cqaASjXn2K /tmp/tmp.3TnXwFRHY3 ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.h64qfRAcDN ++ mktemp + local LAST_ERR=/tmp/tmp.cte7YUZHDa + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp","rs0")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.h64qfRAcDN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ea77c455-48af-4d0b-a922-3c515785af21") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1747055335, 7), "signature" : { "hash" : BinData(0,"3gwBejoySIB6q+CBMVXHod7Ki0s="), "keyId" : NumberLong("7503544153737789463") } }, "operationTime" : Timestamp(1747055335, 1) } bye + cat /tmp/tmp.cte7YUZHDa + rm /tmp/tmp.h64qfRAcDN /tmp/tmp.cte7YUZHDa + return 0 + run_mongos 'sh.enableSharding("myApp1","rs1")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=sh.enableSharding("myApp1","rs1")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.TCJcNFNIS2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.aAiXVZ0rjE ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.TCJcNFNIS2 ++ cat /tmp/tmp.aAiXVZ0rjE ++ rm /tmp/tmp.TCJcNFNIS2 /tmp/tmp.aAiXVZ0rjE ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.PrWnTHbKuN ++ mktemp + local LAST_ERR=/tmp/tmp.X3JP9Lf456 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp1","rs1")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.PrWnTHbKuN Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("dd25b368-02f8-42f8-82b4-4b9d4e17b994") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1747055337, 9), "signature" : { "hash" : BinData(0,"R+2v1HiJ5Cv6YO+r5y+J5MsDbvI="), "keyId" : NumberLong("7503544153737789463") } }, "operationTime" : Timestamp(1747055337, 4) } bye + cat /tmp/tmp.X3JP9Lf456 + rm /tmp/tmp.PrWnTHbKuN /tmp/tmp.X3JP9Lf456 + return 0 + run_mongos 'sh.enableSharding("myApp2","rs2")' clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=sh.enableSharding("myApp2","rs2")' + local uri=clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VSwhwciSOl +++ mktemp ++ local LAST_ERR=/tmp/tmp.sz2IuxBPCz ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.VSwhwciSOl ++ cat /tmp/tmp.sz2IuxBPCz ++ rm /tmp/tmp.VSwhwciSOl /tmp/tmp.sz2IuxBPCz ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.pXmK1ngA05 ++ mktemp + local LAST_ERR=/tmp/tmp.Atxj0S04hG + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''sh.enableSharding("myApp2","rs2")\n'\'' | mongo mongodb://clusterAdmin:clusterAdmin123456@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pXmK1ngA05 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("226c8b7b-8482-411b-a574-f488e700485a") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match { "ok" : 1, "$clusterTime" : { "clusterTime" : Timestamp(1747055340, 8), "signature" : { "hash" : BinData(0,"btEHriLglXOjKgjwX+aIRqqeBjY="), "keyId" : NumberLong("7503544153737789463") } }, "operationTime" : Timestamp(1747055340, 3) } bye + cat /tmp/tmp.Atxj0S04hG + rm /tmp/tmp.pXmK1ngA05 /tmp/tmp.Atxj0S04hG + return 0 + insert_data_mongos 100500 myApp '' 27019 + local data=100500 + local db_name=myApp + local flags= + local port=27019 + run_mongos 'use myApp\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=use myApp\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nS91RTPpXV +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mq2yMfkX47 ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.nS91RTPpXV ++ cat /tmp/tmp.Mq2yMfkX47 ++ rm /tmp/tmp.nS91RTPpXV /tmp/tmp.Mq2yMfkX47 ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.6rUMRTWN30 ++ mktemp + local LAST_ERR=/tmp/tmp.mni5k4r7At + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.6rUMRTWN30 Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("ce2b0944-e991-4545-a77a-ef28c6ed031f") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.mni5k4r7At + rm /tmp/tmp.6rUMRTWN30 /tmp/tmp.mni5k4r7At + return 0 + insert_data_mongos 100500 myApp1 '' 27019 + local data=100500 + local db_name=myApp1 + local flags= + local port=27019 + run_mongos 'use myApp1\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=use myApp1\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ echo .svc.cluster.local ++ awk -F: '{print $2}' + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OWVyBOeDdl +++ mktemp ++ local LAST_ERR=/tmp/tmp.EHCqRN2OdT ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.OWVyBOeDdl ++ cat /tmp/tmp.EHCqRN2OdT ++ rm /tmp/tmp.OWVyBOeDdl /tmp/tmp.EHCqRN2OdT ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.Bh7fEKUkHy ++ mktemp + local LAST_ERR=/tmp/tmp.V7bXToePbv + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp1\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.Bh7fEKUkHy Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("1b74823b-8f9d-445f-a5e6-e15f3b06f12c") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp1 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.V7bXToePbv + rm /tmp/tmp.Bh7fEKUkHy /tmp/tmp.V7bXToePbv + return 0 + insert_data_mongos 100500 myApp2 '' 27019 + local data=100500 + local db_name=myApp2 + local flags= + local port=27019 + run_mongos 'use myApp2\n db.test.insert({ x: 100500 })' myApp:myPass@some-name-mongos.demand-backup-sharded-24520 '' '' '' 27019 + local 'command=use myApp2\n db.test.insert({ x: 100500 })' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pZjXFxVygR +++ mktemp ++ local LAST_ERR=/tmp/tmp.ETZ7CeySfW ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.pZjXFxVygR ++ cat /tmp/tmp.ETZ7CeySfW ++ rm /tmp/tmp.pZjXFxVygR /tmp/tmp.ETZ7CeySfW ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.hMnoLzssxD ++ mktemp + local LAST_ERR=/tmp/tmp.3pLvvcOtHA + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp2\n db.test.insert({ x: 100500 })\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.hMnoLzssxD Percona Server for MongoDB shell version v4.4.29-28 connecting to: mongodb://some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin?compressors=disabled&gssapiServiceName=mongodb Implicit session: session { "id" : UUID("df2114bf-7ef6-4eed-8fb8-2c0de49fbcd9") } Percona Server for MongoDB server version: v7.0.18-11 WARNING: shell and server versions do not match switched to db myApp2 WriteResult({ "nInserted" : 1 }) bye + cat /tmp/tmp.3pLvvcOtHA + rm /tmp/tmp.hMnoLzssxD /tmp/tmp.3pLvvcOtHA + return 0 + minikube_sleep + sleep_time=10 + [[ '' == 1 ]] + compare_mongos_cmd find myApp:myPass@some-name-mongos.demand-backup-sharded-24520 '' '' '' '' 27019 + local command=find + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-24520 + local postfix= + local suffix= + local database=myApp + local collection=test + local port=27019 + /usr/bin/sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' + egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:' + run_mongos 'use myApp\n db.test.find()' myApp:myPass@some-name-mongos.demand-backup-sharded-24520 mongodb '' '' 27019 + local 'command=use myApp\n db.test.find()' + local uri=myApp:myPass@some-name-mongos.demand-backup-sharded-24520 + local driver=mongodb + local suffix=.svc.cluster.local + local mongo_flag= + local port=27019 + local mongo_bin=mongo ++ awk -F: '{print $2}' ++ echo .svc.cluster.local + suffix_port= + [[ -z '' ]] + suffix=.svc.cluster.local:27019 ++ kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yNrfvwxL2K +++ mktemp ++ local LAST_ERR=/tmp/tmp.WCWuuflGzc ++ local exit_status=0 ++ local timeout=4 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 -a -n 1 ']' ++ break ++ cat /tmp/tmp.yNrfvwxL2K ++ cat /tmp/tmp.WCWuuflGzc ++ rm /tmp/tmp.yNrfvwxL2K /tmp/tmp.WCWuuflGzc ++ return 0 + local client_container=psmdb-client-66f577db5f-2mqpq + kubectl_bin exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' ++ mktemp + local LAST_OUT=/tmp/tmp.u6DQRtqM7L ++ mktemp + local LAST_ERR=/tmp/tmp.LrOAD4vY9v + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl exec psmdb-client-66f577db5f-2mqpq -- bash -c 'printf '\''use myApp\n db.test.find()\n'\'' | mongo mongodb://myApp:myPass@some-name-mongos.demand-backup-sharded-24520.svc.cluster.local:27019/admin ' + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.u6DQRtqM7L + cat /tmp/tmp.LrOAD4vY9v + rm /tmp/tmp.u6DQRtqM7L /tmp/tmp.LrOAD4vY9v + return 0 + [[ 0 -eq 0 ]] + diff /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/compare/find.json /tmp/tmp.fmiE2aOTZS/find + wait_backup_agent some-name-rs0-0 + local agent_pod=some-name-rs0-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-0...2025-05-12T13:04:17.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-1 + local agent_pod=some-name-rs0-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-1...2025-05-12T13:04:15.000+0000 I listening for the commands + wait_backup_agent some-name-rs0-2 + local agent_pod=some-name-rs0-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs0-2...2025-05-12T13:04:16.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-0 + local agent_pod=some-name-rs1-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-0...2025-05-12T13:04:19.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-1 + local agent_pod=some-name-rs1-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-1...2025-05-12T13:04:17.000+0000 I listening for the commands + wait_backup_agent some-name-rs1-2 + local agent_pod=some-name-rs1-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs1-2...2025-05-12T13:04:14.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-0 + local agent_pod=some-name-rs2-0 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-0...2025-05-12T13:04:15.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-1 + local agent_pod=some-name-rs2-1 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-1...2025-05-12T13:04:18.000+0000 I listening for the commands + wait_backup_agent some-name-rs2-2 + local agent_pod=some-name-rs2-2 + set +o xtrace waiting for pbm-agent to be ready in some-name-rs2-2...2025-05-12T13:04:17.000+0000 I listening for the commands + backup_name_aws=backup-aws-s3 + backup_name_minio=backup-minio + backup_name_gcp=backup-gcp-cs + backup_name_azure=backup-azure-blob + desc 'run backups' + set +o xtrace ----------------------------------------------------------------------------------- run backups ----------------------------------------------------------------------------------- + run_backup minio + local storage=minio + local backup_name=backup-minio + local type=logical + desc 'run backup backup-minio' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-minio ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-minio" | .spec.storageName = "minio" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/backup-minio.yml ++ mktemp + local LAST_OUT=/tmp/tmp.pRry7CrJMA ++ mktemp + local LAST_ERR=/tmp/tmp.eUUEzBrLrf + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.pRry7CrJMA perconaservermongodbbackup.psmdb.percona.com/backup-minio created + cat /tmp/tmp.eUUEzBrLrf + rm /tmp/tmp.pRry7CrJMA /tmp/tmp.eUUEzBrLrf + return 0 + '[' -z '' ']' + run_backup aws-s3 + local storage=aws-s3 + local backup_name=backup-aws-s3 + local type=logical + desc 'run backup backup-aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-aws-s3 ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-aws-s3" | .spec.storageName = "aws-s3" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/backup-aws-s3.yml ++ mktemp + local LAST_OUT=/tmp/tmp.wBHilKx1jF ++ mktemp + local LAST_ERR=/tmp/tmp.4cJGQ0HUT2 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.wBHilKx1jF perconaservermongodbbackup.psmdb.percona.com/backup-aws-s3 created + cat /tmp/tmp.4cJGQ0HUT2 + rm /tmp/tmp.wBHilKx1jF /tmp/tmp.4cJGQ0HUT2 + return 0 + run_backup gcp-cs + local storage=gcp-cs + local backup_name=backup-gcp-cs + local type=logical + desc 'run backup backup-gcp-cs' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-gcp-cs ----------------------------------------------------------------------------------- + yq eval '.metadata.name = "backup-gcp-cs" | .spec.storageName = "gcp-cs" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/backup-gcp-cs.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.sI3iAlGZpP ++ mktemp + local LAST_ERR=/tmp/tmp.d6NTGopQE0 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.sI3iAlGZpP perconaservermongodbbackup.psmdb.percona.com/backup-gcp-cs created + cat /tmp/tmp.d6NTGopQE0 + rm /tmp/tmp.sI3iAlGZpP /tmp/tmp.d6NTGopQE0 + return 0 + run_backup azure-blob + local storage=azure-blob + local backup_name=backup-azure-blob + local type=logical + desc 'run backup backup-azure-blob' + set +o xtrace ----------------------------------------------------------------------------------- run backup backup-azure-blob ----------------------------------------------------------------------------------- + kubectl_bin apply -f - + yq eval '.metadata.name = "backup-azure-blob" | .spec.storageName = "azure-blob" | .spec.type = "logical"' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-1904/e2e-tests/demand-backup-sharded/conf/backup-azure-blob.yml ++ mktemp + local LAST_OUT=/tmp/tmp.cuvderz9fr ++ mktemp + local LAST_ERR=/tmp/tmp.s1ll61Be26 + local exit_status=0 + local timeout=4 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 -a -n 1 ']' + break + cat /tmp/tmp.cuvderz9fr perconaservermongodbbackup.psmdb.percona.com/backup-azure-blob created + cat /tmp/tmp.s1ll61Be26 + rm /tmp/tmp.cuvderz9fr /tmp/tmp.s1ll61Be26 + return 0 + wait_backup backup-aws-s3 + local backup_name=backup-aws-s3 + local target_state=ready + set +o xtrace waiting for backup-aws-s3 to reach ready state......................2025-05-12T13:09:51.532Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "9c98ab99-cfdc-4187-8483-214b0fe4a282", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:51.533Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "9c98ab99-cfdc-4187-8483-214b0fe4a282", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-12T13:09:51.533Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "9c98ab99-cfdc-4187-8483-214b0fe4a282", "backup": "backup-minio", "status": "running"} 2025-05-12T13:09:51.534Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "9c98ab99-cfdc-4187-8483-214b0fe4a282"} 2025-05-12T13:09:53.150Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "73db702c-4632-4e6c-a143-1489beefab4f"} 2025-05-12T13:09:53.230Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "73db702c-4632-4e6c-a143-1489beefab4f", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:53.230Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "73db702c-4632-4e6c-a143-1489beefab4f", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-12T13:09:53.230Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "73db702c-4632-4e6c-a143-1489beefab4f", "backup": "backup-minio", "status": "running"} 2025-05-12T13:09:53.231Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "73db702c-4632-4e6c-a143-1489beefab4f"} 2025-05-12T13:09:53.863Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355"} 2025-05-12T13:09:53.942Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:53.942Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355", "currentJob": {"Name":"backup-minio","Type":0}} 2025-05-12T13:09:54.032Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355", "backup": "backup-minio", "pbmName": "2025-05-12T13:09:22Z", "meta": {"type":"logical","opid":"6821f3026b3565c455b08ac1","name":"2025-05-12T13:09:22Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs1","backup_name":"2025-05-12T13:09:22Z/rs1/metadata.json","oplog_name":"2025-05-12T13:09:22Z/rs1/oplog","start_ts":1747055363,"status":"done","last_transition_ts":1747055382,"first_write_ts":{"T":1747055362,"I":2},"last_write_ts":{"T":1747055367,"I":9},"node":"some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055363,"status":"running"},{"timestamp":1747055367,"status":"dumpDone"},{"timestamp":1747055382,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs2","backup_name":"2025-05-12T13:09:22Z/rs2/metadata.json","oplog_name":"2025-05-12T13:09:22Z/rs2/oplog","start_ts":1747055364,"status":"done","last_transition_ts":1747055384,"first_write_ts":{"T":1747055347,"I":4},"last_write_ts":{"T":1747055367,"I":53},"node":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055364,"status":"running"},{"timestamp":1747055367,"status":"dumpDone"},{"timestamp":1747055384,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs0","backup_name":"2025-05-12T13:09:22Z/rs0/metadata.json","oplog_name":"2025-05-12T13:09:22Z/rs0/oplog","start_ts":1747055364,"status":"done","last_transition_ts":1747055381,"first_write_ts":{"T":1747055361,"I":1},"last_write_ts":{"T":1747055367,"I":58},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055364,"status":"running"},{"timestamp":1747055367,"status":"dumpDone"},{"timestamp":1747055381,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-12T13:09:22Z/cfg/metadata.json","oplog_name":"2025-05-12T13:09:22Z/cfg/oplog","start_ts":1747055363,"status":"done","iscs":true,"last_transition_ts":1747055388,"first_write_ts":{"T":1747055363,"I":52},"last_write_ts":{"T":1747055368,"I":1},"node":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","conditions":[{"timestamp":1747055363,"status":"running"},{"timestamp":1747055367,"status":"dumpDone"},{"timestamp":1747055388,"status":"done"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"name":"minio","profile":true,"type":"s3","s3":{"region":"us-east-1","endpointUrl":"http://minio-service:9000/","forcePathStyle":true,"bucket":"operator-testing","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":157851,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747055363,"last_transition_ts":1747055389,"first_write_ts":{"T":1747055363,"I":52},"last_write_ts":{"T":1747055367,"I":9},"hb":{"T":1747055388,"I":9},"status":"done","conditions":[{"timestamp":1747055363,"status":"starting"},{"timestamp":1747055366,"status":"running"},{"timestamp":1747055368,"status":"dumpDone"},{"timestamp":1747055389,"status":"done"}],"n":[{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"],"ack":"some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-12T13:09:54.033Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355", "previous": "running", "current": "ready"} 2025-05-12T13:09:54.050Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "3d50b1a6-2c84-4e6f-8dda-70c31a126355"} 2025-05-12T13:09:54.050Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "b95fa043-ba6f-40b4-bd92-4f2388712ae4"} 2025-05-12T13:09:54.050Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "b95fa043-ba6f-40b4-bd92-4f2388712ae4"} 2025-05-12T13:09:54.835Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938"} 2025-05-12T13:09:54.926Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:54.926Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-12T13:09:55.030Z INFO Starting backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938", "backup": "backup-aws-s3", "storage": "aws-s3"} 2025-05-12T13:09:55.030Z INFO Sending backup command {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938", "backup": "backup-aws-s3", "storage": "aws-s3", "backupCmd": "backup [name: 2025-05-12T13:09:55Z, compression: gzip (level: default)] ", "profile": ""} 2025-05-12T13:09:55.038Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938", "previous": "waiting", "current": "requested"} 2025-05-12T13:09:55.057Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "de9de7c2-857c-495f-b172-a471c8815938"} 2025-05-12T13:09:55.057Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "171d5cb6-9f22-40fa-9770-39df21effcbe"} 2025-05-12T13:09:55.137Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "171d5cb6-9f22-40fa-9770-39df21effcbe", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:55.137Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "171d5cb6-9f22-40fa-9770-39df21effcbe", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-12T13:09:55.220Z INFO Waiting for backup metadata {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "171d5cb6-9f22-40fa-9770-39df21effcbe", "pbmName": "2025-05-12T13:09:55Z", "backup": "backup-aws-s3"} 2025-05-12T13:09:55.221Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "171d5cb6-9f22-40fa-9770-39df21effcbe"} 2025-05-12T13:09:56.535Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "b2d30089-2157-4171-8773-bab3271ca519"} 2025-05-12T13:09:56.617Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "b2d30089-2157-4171-8773-bab3271ca519", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:56.617Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "b2d30089-2157-4171-8773-bab3271ca519", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-12T13:09:56.617Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "b2d30089-2157-4171-8773-bab3271ca519", "backup": "backup-aws-s3", "status": "requested"} 2025-05-12T13:09:56.618Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "b2d30089-2157-4171-8773-bab3271ca519"} 2025-05-12T13:09:58.231Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "4c4e44af-6c05-4122-87ae-904b1bc30968"} 2025-05-12T13:09:58.319Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "4c4e44af-6c05-4122-87ae-904b1bc30968", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:09:58.319Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "4c4e44af-6c05-4122-87ae-904b1bc30968", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-12T13:09:58.319Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "4c4e44af-6c05-4122-87ae-904b1bc30968", "backup": "backup-aws-s3", "status": "requested"} 2025-05-12T13:09:58.320Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "4c4e44af-6c05-4122-87ae-904b1bc30968"} 2025-05-12T13:09:59.051Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "ec7e6331-f123-4650-9e64-6f17bdc57901"} 2025-05-12T13:09:59.052Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-minio","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-minio", "reconcileID": "ec7e6331-f123-4650-9e64-6f17bdc57901"} 2025-05-12T13:10:00.058Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3"} 2025-05-12T13:10:00.179Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:00.179Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-12T13:10:00.315Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3", "backup": "backup-aws-s3", "pbmName": "2025-05-12T13:09:55Z", "meta": {"type":"logical","opid":"6821f3236b3565c455b08b44","name":"2025-05-12T13:09:55Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-12T13:09:55Z/rs0/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs0/oplog","start_ts":1747055396,"status":"running","last_transition_ts":1747055396,"first_write_ts":{"T":1747055381,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs1","backup_name":"2025-05-12T13:09:55Z/rs1/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs1/oplog","start_ts":1747055396,"status":"running","last_transition_ts":1747055396,"first_write_ts":{"T":1747055393,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs2","backup_name":"2025-05-12T13:09:55Z/rs2/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs2/oplog","start_ts":1747055396,"status":"running","last_transition_ts":1747055396,"first_write_ts":{"T":1747055384,"I":10},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-12T13:09:55Z/cfg/metadata.json","oplog_name":"2025-05-12T13:09:55Z/cfg/oplog","start_ts":1747055396,"status":"running","iscs":true,"last_transition_ts":1747055396,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":0,"I":0},"node":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747055395,"last_transition_ts":1747055399,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747055396,"I":8},"status":"running","conditions":[{"timestamp":1747055395,"status":"starting"},{"timestamp":1747055399,"status":"running"}],"n":[{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-12T13:10:00.319Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3", "previous": "requested", "current": "running"} 2025-05-12T13:10:00.337Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "d8510d96-b557-432e-91c6-8798a0bf01a3"} 2025-05-12T13:10:00.337Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "f9e99a7c-99da-40ef-a803-9e082a44829c"} 2025-05-12T13:10:00.436Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "f9e99a7c-99da-40ef-a803-9e082a44829c", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:00.436Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "f9e99a7c-99da-40ef-a803-9e082a44829c", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-12T13:10:00.528Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "f9e99a7c-99da-40ef-a803-9e082a44829c", "backup": "backup-aws-s3", "pbmName": "2025-05-12T13:09:55Z", "meta": {"type":"logical","opid":"6821f3236b3565c455b08b44","name":"2025-05-12T13:09:55Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-12T13:09:55Z/rs0/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs0/oplog","start_ts":1747055396,"status":"dumpDone","last_transition_ts":1747055400,"first_write_ts":{"T":1747055381,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"},{"timestamp":1747055400,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs1","backup_name":"2025-05-12T13:09:55Z/rs1/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs1/oplog","start_ts":1747055396,"status":"running","last_transition_ts":1747055396,"first_write_ts":{"T":1747055393,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs2","backup_name":"2025-05-12T13:09:55Z/rs2/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs2/oplog","start_ts":1747055396,"status":"running","last_transition_ts":1747055396,"first_write_ts":{"T":1747055384,"I":10},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-12T13:09:55Z/cfg/metadata.json","oplog_name":"2025-05-12T13:09:55Z/cfg/oplog","start_ts":1747055396,"status":"running","iscs":true,"last_transition_ts":1747055396,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":0,"I":0},"node":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","conditions":[{"timestamp":1747055396,"status":"running"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747055395,"last_transition_ts":1747055399,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747055396,"I":8},"status":"running","conditions":[{"timestamp":1747055395,"status":"starting"},{"timestamp":1747055399,"status":"running"}],"n":[{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-12T13:10:00.530Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "f9e99a7c-99da-40ef-a803-9e082a44829c"} 2025-05-12T13:10:01.619Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "31b25f64-1d09-4be1-bb3d-dbe14b0f088d"} 2025-05-12T13:10:01.703Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "31b25f64-1d09-4be1-bb3d-dbe14b0f088d", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:01.703Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "31b25f64-1d09-4be1-bb3d-dbe14b0f088d", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-12T13:10:01.703Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "31b25f64-1d09-4be1-bb3d-dbe14b0f088d", "backup": "backup-aws-s3", "status": "running"} 2025-05-12T13:10:01.704Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "31b25f64-1d09-4be1-bb3d-dbe14b0f088d"} 2025-05-12T13:10:03.322Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "46a4c03b-8e67-48a2-9930-48a2c89cc3f9"} 2025-05-12T13:10:03.454Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "46a4c03b-8e67-48a2-9930-48a2c89cc3f9", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:03.455Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "46a4c03b-8e67-48a2-9930-48a2c89cc3f9", "currentJob": {"Name":"backup-azure-blob","Type":0}} 2025-05-12T13:10:03.455Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "46a4c03b-8e67-48a2-9930-48a2c89cc3f9", "backup": "backup-aws-s3", "status": "running"} 2025-05-12T13:10:03.457Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "46a4c03b-8e67-48a2-9930-48a2c89cc3f9"} 2025-05-12T13:10:04.897Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "some-name", "reconcileID": "8fc0e677-014d-4126-ab7d-75e89decab12", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: handshake failure: (ShutdownInProgress) Mongos is in quiesce mode and will shut down", "errorVerbose": "connection() error occurred during connection handshake: handshake failure: (ShutdownInProgress) Mongos is in quiesce mode and will shut down\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:544\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:544\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:05.338Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "8474048e-0408-4063-b0f7-fb63f71867ea"} 2025-05-12T13:10:05.462Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "8474048e-0408-4063-b0f7-fb63f71867ea", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:05.462Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "8474048e-0408-4063-b0f7-fb63f71867ea", "currentJob": {"Name":"backup-aws-s3","Type":0}} 2025-05-12T13:10:05.580Z DEBUG backupStatus Got backup meta {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "8474048e-0408-4063-b0f7-fb63f71867ea", "backup": "backup-aws-s3", "pbmName": "2025-05-12T13:09:55Z", "meta": {"type":"logical","opid":"6821f3236b3565c455b08b44","name":"2025-05-12T13:09:55Z","shardRemap":{"cfg":"config"},"replsets":[{"name":"rs0","backup_name":"2025-05-12T13:09:55Z/rs0/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs0/oplog","start_ts":1747055396,"status":"dumpDone","last_transition_ts":1747055400,"first_write_ts":{"T":1747055381,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"},{"timestamp":1747055400,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs1","backup_name":"2025-05-12T13:09:55Z/rs1/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs1/oplog","start_ts":1747055396,"status":"dumpDone","last_transition_ts":1747055400,"first_write_ts":{"T":1747055393,"I":3},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"},{"timestamp":1747055400,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"rs2","backup_name":"2025-05-12T13:09:55Z/rs2/metadata.json","oplog_name":"2025-05-12T13:09:55Z/rs2/oplog","start_ts":1747055396,"status":"dumpDone","last_transition_ts":1747055400,"first_write_ts":{"T":1747055384,"I":10},"last_write_ts":{"T":0,"I":0},"node":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","conditions":[{"timestamp":1747055396,"status":"running"},{"timestamp":1747055400,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"},{"name":"cfg","backup_name":"2025-05-12T13:09:55Z/cfg/metadata.json","oplog_name":"2025-05-12T13:09:55Z/cfg/oplog","start_ts":1747055396,"status":"dumpDone","iscs":true,"last_transition_ts":1747055400,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":1747055401,"I":7},"node":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","conditions":[{"timestamp":1747055396,"status":"running"},{"timestamp":1747055400,"status":"dumpDone"}],"pbm_version":"2.9.1","mongo_version":"7.0.18-11"}],"compression":"gzip","store":{"type":"s3","s3":{"region":"us-east-1","endpointUrl":"","forcePathStyle":true,"bucket":"operator-testing","prefix":"psmdb-demand-backup-sharded","maxUploadParts":10000,"storageClass":"STANDARD","insecureSkipTLSVerify":false}},"size":0,"mongodb_version":"7.0.18-11","fcv":"7.0","start_ts":1747055395,"last_transition_ts":1747055401,"first_write_ts":{"T":1747055396,"I":8},"last_write_ts":{"T":1,"I":1},"hb":{"T":1747055401,"I":3},"status":"dumpDone","conditions":[{"timestamp":1747055395,"status":"starting"},{"timestamp":1747055399,"status":"running"},{"timestamp":1747055401,"status":"dumpDone"}],"n":[{"rs":"rs2","n":["some-name-rs2-2.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs2-1.some-name-rs2.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"cfg","n":["some-name-cfg-1.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018","some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"],"ack":"some-name-cfg-2.some-name-cfg.demand-backup-sharded-24520.svc.cluster.local:27018"},{"rs":"rs1","n":["some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs1-1.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs1-2.some-name-rs1.demand-backup-sharded-24520.svc.cluster.local:27016"},{"rs":"rs0","n":["some-name-rs0-1.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016","some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"],"ack":"some-name-rs0-2.some-name-rs0.demand-backup-sharded-24520.svc.cluster.local:27016"}],"pbm_version":"2.9.1","balancer":"full"}} 2025-05-12T13:10:05.582Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "8474048e-0408-4063-b0f7-fb63f71867ea"} 2025-05-12T13:10:06.706Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "59e5e9c1-4781-4f42-b35c-56ec23954bf1"} 2025-05-12T13:10:06.813Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "59e5e9c1-4781-4f42-b35c-56ec23954bf1", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:06.813Z DEBUG Checking for active jobs {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "59e5e9c1-4781-4f42-b35c-56ec23954bf1", "currentJob": {"Name":"backup-gcp-cs","Type":0}} 2025-05-12T13:10:06.814Z INFO Waiting for backup to complete {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "59e5e9c1-4781-4f42-b35c-56ec23954bf1", "backup": "backup-aws-s3", "status": "running"} 2025-05-12T13:10:06.815Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "59e5e9c1-4781-4f42-b35c-56ec23954bf1"} 2025-05-12T13:10:07.370Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "some-name", "reconcileID": "8fc0e677-014d-4126-ab7d-75e89decab12", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: handshake failure: (ShutdownInProgress) Mongos is in quiesce mode and will shut down", "errorVerbose": "failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: handshake failure: (ShutdownInProgress) Mongos is in quiesce mode and will shut down\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:08.459Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1"} 2025-05-12T13:10:08.591Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:08.592Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1", "backup": "backup-azure-blob", "error": "failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:138 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:08.592Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1", "previous": "waiting", "current": "error"} 2025-05-12T13:10:08.611Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1"} 2025-05-12T13:10:08.611Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1"} 2025-05-12T13:10:08.611Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "357d8ef1-b078-4348-9fb4-e5f6ea1549f1", "error": "reconcile backup: failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:08.612Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "d95d48ba-17a6-428d-96db-6dacdb73c9a4"} 2025-05-12T13:10:08.612Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "d95d48ba-17a6-428d-96db-6dacdb73c9a4"} 2025-05-12T13:10:08.617Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "a92e493d-8dfb-4f44-af84-f8d307e5bf90"} 2025-05-12T13:10:08.617Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-azure-blob","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-azure-blob", "reconcileID": "a92e493d-8dfb-4f44-af84-f8d307e5bf90"} 2025-05-12T13:10:09.064Z ERROR failed to reconcile cluster {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "some-name", "reconcileID": "430aca9d-a354-4e75-b4c2-48e8b5308780", "replset": "rs0", "error": "failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: dial tcp 34.118.235.41:27019: connect: connection refused", "errorVerbose": "connection() error occurred during connection handshake: dial tcp 34.118.235.41:27019: connect: connection refused\nping mongo\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo.Dial\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo/mongo.go:123\ngithub.com/percona/percona-server-mongodb-operator/pkg/psmdb.MongosClient\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/psmdb/client.go:85\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*mongoClientProvider).Mongos\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:47\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).mongosClientWithRole\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/connections.go:64\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:196\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:544\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to get mongos connection\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileCluster\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/mgo.go:198\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:544\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).reconcileReplsets /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:546 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:418 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:09.907Z DEBUG Replset is not ready {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "some-name", "reconcileID": "430aca9d-a354-4e75-b4c2-48e8b5308780", "liveMembers": 2, "pods": 3} 2025-05-12T13:10:09.975Z ERROR Reconciler error {"controller": "psmdb-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDB", "PerconaServerMongoDB": {"name":"some-name","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "some-name", "reconcileID": "430aca9d-a354-4e75-b4c2-48e8b5308780", "error": "reconcile statefulsets: failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: dial tcp 34.118.235.41:27019: connect: connection refused", "errorVerbose": "failed to get mongos connection: ping mongo: connection() error occurred during connection handshake: dial tcp 34.118.235.41:27019: connect: connection refused\nreconcile statefulsets\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb.(*ReconcilePerconaServerMongoDB).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodb/psmdb_controller.go:420\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:10.584Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1"} 2025-05-12T13:10:10.684Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:10.685Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1", "backup": "backup-aws-s3", "error": "failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:138 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:10.685Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1", "previous": "running", "current": "error"} 2025-05-12T13:10:10.783Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1"} 2025-05-12T13:10:10.783Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1"} 2025-05-12T13:10:10.783Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "b851ad55-3c02-4162-a24d-473b7910aca1", "error": "reconcile backup: failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:10.784Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "ad9b5da1-6086-4d33-9759-1b609d352574"} 2025-05-12T13:10:10.784Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "ad9b5da1-6086-4d33-9759-1b609d352574"} 2025-05-12T13:10:10.789Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "c0549830-9662-4f71-8993-67b82ea1f334"} 2025-05-12T13:10:10.789Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-aws-s3","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-aws-s3", "reconcileID": "c0549830-9662-4f71-8993-67b82ea1f334"} 2025-05-12T13:10:11.816Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106"} 2025-05-12T13:10:11.945Z DEBUG checking if backup is allowed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106", "cluster": "some-name", "namespace": "demand-backup-sharded-24520"} 2025-05-12T13:10:11.946Z ERROR failed to make backup {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106", "backup": "backup-gcp-cs", "error": "failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile.func1 /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:138 github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile /go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:11.946Z INFO Backup state changed {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106", "previous": "waiting", "current": "error"} 2025-05-12T13:10:12.065Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106"} 2025-05-12T13:10:12.065Z INFO Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106"} 2025-05-12T13:10:12.065Z ERROR Reconciler error {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "2fb667f0-ea90-4a47-b373-fef7913a3106", "error": "reconcile backup: failed to run backup: spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error", "errorVerbose": "spec.unsafeFlags.backupIfUnhealthy must be true to run backup on cluster with status error\ngithub.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1.(*PerconaServerMongoDB).CanBackup\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1/psmdb_types.go:1255\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:221\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to run backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:222\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:199\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nreconcile backup\ngithub.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup.(*ReconcilePerconaServerMongoDBBackup).Reconcile\n\t/go/src/github.com/percona/percona-server-mongodb-operator/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:334\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:347 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:294 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.20.4/pkg/internal/controller/controller.go:255 2025-05-12T13:10:12.066Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "470bb8de-01ec-4433-81f8-377ea95689de"} 2025-05-12T13:10:12.066Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "470bb8de-01ec-4433-81f8-377ea95689de"} 2025-05-12T13:10:12.071Z DEBUG Reconciling {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "bbd27292-3b50-45a4-8c41-c6fcbade79b6"} 2025-05-12T13:10:12.071Z DEBUG Reconcile finished {"controller": "psmdbbackup-controller", "controllerGroup": "psmdb.percona.com", "controllerKind": "PerconaServerMongoDBBackup", "PerconaServerMongoDBBackup": {"name":"backup-gcp-cs","namespace":"demand-backup-sharded-24520"}, "namespace": "demand-backup-sharded-24520", "name": "backup-gcp-cs", "reconcileID": "bbd27292-3b50-45a4-8c41-c6fcbade79b6"} NAME CLUSTER STORAGE DESTINATION TYPE STATUS COMPLETED AGE backup-aws-s3 some-name aws-s3 s3://operator-testing/psmdb-demand-backup-sharded/2025-05-12T13:09:55Z logical error 54s backup-azure-blob some-name azure-blob error 51s backup-gcp-cs some-name gcp-cs error 53s backup-minio some-name minio s3://operator-testing/2025-05-12T13:09:22Z logical ready 29s 56s Backup object psmdb-backup/backup-aws-s3 is in error state. something went wrong with operator or kubernetes cluster