Log: /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/logs/demand-backup-physical-aws.log
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.32) exceeds the supported minor version skew of +/-1
+ '[' -n '' ']'
+ create_infra demand-backup-physical-aws-13657
+ local ns=demand-backup-physical-aws-13657
+ [[ 1 == 1 ]]
+ delete_crd
+ desc 'get and delete old CRDs and RBAC'
+ set +o xtrace


-----------------------------------------------------------------------------------
get and delete old CRDs and RBAC
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml --ignore-not-found --wait=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.pZTfuDl9bk
++ mktemp
+ local LAST_ERR=/tmp/tmp.v7wUmOFpWA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml --ignore-not-found --wait=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.pZTfuDl9bk
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbbackups.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbrestores.psmdb.percona.com" deleted
customresourcedefinition.apiextensions.k8s.io "perconaservermongodbs.psmdb.percona.com" deleted
+ cat /tmp/tmp.v7wUmOFpWA
+ rm /tmp/tmp.pZTfuDl9bk /tmp/tmp.v7wUmOFpWA
+ return 0
++ yq eval .metadata.name /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
++ grep -v '\-\-\-'
grep: warning: stray \ before -
grep: warning: stray \ before -
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbbackups.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbbackups.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ kubectl patch perconaservermongodbbackups.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbbackups"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.QmPZ1a8B3p
++ mktemp
+ local LAST_ERR=/tmp/tmp.jrCERGHDgb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbbackups.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QmPZ1a8B3p
+ cat /tmp/tmp.jrCERGHDgb
+ rm /tmp/tmp.QmPZ1a8B3p /tmp/tmp.jrCERGHDgb
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbrestores.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbrestores.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ kubectl patch perconaservermongodbrestores.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: the server doesn't have a resource type "perconaservermongodbrestores"
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.LM3QxvaxJq
++ mktemp
+ local LAST_ERR=/tmp/tmp.r5TKW0BnqS
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbrestores.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.LM3QxvaxJq
+ cat /tmp/tmp.r5TKW0BnqS
+ rm /tmp/tmp.LM3QxvaxJq /tmp/tmp.r5TKW0BnqS
+ return 0
+ for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-')
+ kubectl get perconaservermongodbs.psmdb.percona.com --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch perconaservermongodbs.psmdb.percona.com -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch perconaservermongodbs.psmdb.percona.com -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin wait --for=delete crd perconaservermongodbs.psmdb.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.QDFwb1Jw4Q
++ mktemp
+ local LAST_ERR=/tmp/tmp.7QjCsBF1EK
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete crd perconaservermongodbs.psmdb.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.QDFwb1Jw4Q
+ cat /tmp/tmp.7QjCsBF1EK
+ rm /tmp/tmp.QDFwb1Jw4Q /tmp/tmp.7QjCsBF1EK
+ return 0
+ local rbac_yaml=rbac.yaml
+ '[' -n psmdb-operator ']'
+ rbac_yaml=cw-rbac.yaml
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml --ignore-not-found
++ mktemp
+ local LAST_OUT=/tmp/tmp.BlOyLjZjQF
++ mktemp
+ local LAST_ERR=/tmp/tmp.G9UXfHAHnb
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.BlOyLjZjQF
clusterrole.rbac.authorization.k8s.io "percona-server-mongodb-operator" deleted
clusterrolebinding.rbac.authorization.k8s.io "service-account-percona-server-mongodb-operator" deleted
+ cat /tmp/tmp.G9UXfHAHnb
+ rm /tmp/tmp.BlOyLjZjQF /tmp/tmp.G9UXfHAHnb
+ return 0
+ check_crd_for_deletion PR-2269-20e73be3
+ local git_tag=PR-2269-20e73be3
++ curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/PR-2269-20e73be3/deploy/crd.yaml
++ yq eval .metadata.name
++ /usr/sbin/sed s/---//g
++ /usr/sbin/sed ':a;N;$!ba;s/\n/ /g'
+ for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g')
++ kubectl_bin get crd/null -o 'jsonpath={.status.conditions[-1].type}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JyULbWxopA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ufuezLOShl
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.JyULbWxopA
++ cat /tmp/tmp.ufuezLOShl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 0
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.JyULbWxopA
++ cat /tmp/tmp.ufuezLOShl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 4
++ for i in $(seq 0 2)
++ set +e
++ kubectl get crd/null -o 'jsonpath={.status.conditions[-1].type}'
++ exit_status=1
++ set -e
++ '[' 1 '!=' 0 -a -n 1 ']'
++ cat /tmp/tmp.JyULbWxopA
++ cat /tmp/tmp.ufuezLOShl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ sleep 8
++ cat /tmp/tmp.JyULbWxopA
++ cat /tmp/tmp.ufuezLOShl
Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io "null" not found
++ rm /tmp/tmp.JyULbWxopA /tmp/tmp.ufuezLOShl
++ return 1
+ [[ '' == Terminating ]]
+ '[' -n psmdb-operator ']'
+ create_namespace psmdb-operator
+ local namespace=psmdb-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ xargs kubectl delete ns
++ mktemp
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces psmdb-operator'
+ set +o xtrace
+ local LAST_OUT=/tmp/tmp.4Phv5uUIus


-----------------------------------------------------------------------------------
cleaned up old namespaces psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace psmdb-operator --ignore-not-found
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.fdR8VjV3ex
+ local LAST_ERR=/tmp/tmp.WTNWwrWlr9
+ local exit_status=0
+ local timeout=4
++ mktemp
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
+ local LAST_ERR=/tmp/tmp.fc4hRq9RLA
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace psmdb-operator --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.4Phv5uUIus
+ cat /tmp/tmp.WTNWwrWlr9
+ rm /tmp/tmp.4Phv5uUIus /tmp/tmp.WTNWwrWlr9
+ return 0
namespace "demand-backup-physical-aws-8390" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.fdR8VjV3ex
namespace "psmdb-operator" deleted
+ cat /tmp/tmp.fc4hRq9RLA
+ rm /tmp/tmp.fdR8VjV3ex /tmp/tmp.fc4hRq9RLA
+ return 0
+ kubectl_bin wait --for=delete namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MKrCsJrkfW
++ mktemp
+ local LAST_ERR=/tmp/tmp.RchXjBui01
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MKrCsJrkfW
+ cat /tmp/tmp.RchXjBui01
+ rm /tmp/tmp.MKrCsJrkfW /tmp/tmp.RchXjBui01
+ return 0
+ desc 'create namespace psmdb-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace psmdb-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.VmWehIEfgY
++ mktemp
+ local LAST_ERR=/tmp/tmp.UWP7qDsZQx
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.VmWehIEfgY
namespace/psmdb-operator created
+ cat /tmp/tmp.UWP7qDsZQx
+ rm /tmp/tmp.VmWehIEfgY /tmp/tmp.UWP7qDsZQx
+ return 0
+ set_kube_ctx psmdb-operator
+ local namespace=psmdb-operator
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.anIlU7dxRp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8gjzVbdkbh
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.anIlU7dxRp
++ cat /tmp/tmp.8gjzVbdkbh
++ rm /tmp/tmp.anIlU7dxRp /tmp/tmp.8gjzVbdkbh
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1 --namespace=psmdb-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jyRafkuxDH
++ mktemp
+ local LAST_ERR=/tmp/tmp.SAl98L9EN2
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1 --namespace=psmdb-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.jyRafkuxDH
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1" modified.
+ cat /tmp/tmp.SAl98L9EN2
+ rm /tmp/tmp.jyRafkuxDH /tmp/tmp.SAl98L9EN2
+ return 0
+ deploy_operator
+ desc 'start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PSMDB operator: docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3
-----------------------------------------------------------------------------------

+ local cr_file
+ '[' -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/crd.yaml ']'
+ cr_file=/mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.AGhu1uxuvH
++ mktemp
+ local LAST_ERR=/tmp/tmp.CHBjN6Fdok
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AGhu1uxuvH
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbbackups.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbrestores.psmdb.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaservermongodbs.psmdb.percona.com serverside-applied
+ cat /tmp/tmp.CHBjN6Fdok
+ rm /tmp/tmp.AGhu1uxuvH /tmp/tmp.CHBjN6Fdok
+ return 0
+ '[' -n psmdb-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=psmdb-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: psmdb-operator^'
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.1VHBUNQHnO
++ mktemp
+ local LAST_ERR=/tmp/tmp.O6KDSPinP4
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.1VHBUNQHnO
clusterrole.rbac.authorization.k8s.io/percona-server-mongodb-operator created
serviceaccount/percona-server-mongodb-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-server-mongodb-operator created
+ cat /tmp/tmp.O6KDSPinP4
+ rm /tmp/tmp.1VHBUNQHnO /tmp/tmp.O6KDSPinP4
+ return 0
+ yq eval $'\n\t\t\t(.spec.template.spec.containers[].image = "docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3") |\n\t\t\t((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |\n\t\t\t((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/deploy/cw-operator.yaml
+ kubectl_bin apply -n psmdb-operator -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nOE4cttykn
++ mktemp
+ local LAST_ERR=/tmp/tmp.W3yy85lOoo
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -n psmdb-operator -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.nOE4cttykn
deployment.apps/percona-server-mongodb-operator created
+ cat /tmp/tmp.W3yy85lOoo
+ rm /tmp/tmp.nOE4cttykn /tmp/tmp.W3yy85lOoo
+ return 0
+ sleep 20
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4yOD6YiBLy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3MTv1kFgNu
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.4yOD6YiBLy
++ cat /tmp/tmp.3MTv1kFgNu
++ rm /tmp/tmp.4yOD6YiBLy /tmp/tmp.3MTv1kFgNu
++ return 0
+ wait_operator_pod percona-server-mongodb-operator-dfddcf789-85gdc
+ local pod=percona-server-mongodb-operator-dfddcf789-85gdc
+ set +o xtrace
waiting for pod/percona-server-mongodb-operator-dfddcf789-85gdc to be ready.OK
+ echo 'Print operator info from log'
Print operator info from log
+ grep 'Manager starting up'
++ get_operator_pod
++ kubectl_bin get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0N1rHteFho
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4d1zXrePgW
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl get pods --selector=name=percona-server-mongodb-operator -o 'jsonpath={.items[].metadata.name}' -n psmdb-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.0N1rHteFho
++ cat /tmp/tmp.4d1zXrePgW
++ rm /tmp/tmp.0N1rHteFho /tmp/tmp.4d1zXrePgW
++ return 0
+ kubectl_bin logs -n psmdb-operator percona-server-mongodb-operator-dfddcf789-85gdc
++ mktemp
+ local LAST_OUT=/tmp/tmp.Lo69B9erCq
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dq3QKMZRY5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl logs -n psmdb-operator percona-server-mongodb-operator-dfddcf789-85gdc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Lo69B9erCq
+ cat /tmp/tmp.Dq3QKMZRY5
+ rm /tmp/tmp.Lo69B9erCq /tmp/tmp.Dq3QKMZRY5
+ return 0
2026-04-23T09:53:53.599Z	INFO	setup	Manager starting up	{"gitCommit": "20e73be336bc9107fc60a16452f9669066123774", "gitBranch": "PR-2269-20e73be3", "buildTime": "", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
+ create_namespace demand-backup-physical-aws-13657
+ local namespace=demand-backup-physical-aws-13657
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ desc 'destroy chaos-mesh'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy chaos-mesh
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-physical-aws-13657'
+ awk '{print$1}'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-physical-aws-13657
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-physical-aws-13657 --ignore-not-found
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.AUGyeMvxyB
++ mktemp
+ local LAST_ERR=/tmp/tmp.7uUfrW9ZEV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ local LAST_OUT=/tmp/tmp.cvp6rXgJHZ
+ for i in $(seq 0 2)
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.DXqxxVGaeg
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl delete namespace demand-backup-physical-aws-13657 --ignore-not-found
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.AUGyeMvxyB
+ cat /tmp/tmp.7uUfrW9ZEV
+ rm /tmp/tmp.AUGyeMvxyB /tmp/tmp.7uUfrW9ZEV
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.cvp6rXgJHZ
+ cat /tmp/tmp.DXqxxVGaeg
+ rm /tmp/tmp.cvp6rXgJHZ /tmp/tmp.DXqxxVGaeg
+ return 0
+ kubectl_bin wait --for=delete namespace demand-backup-physical-aws-13657
++ mktemp
+ local LAST_OUT=/tmp/tmp.uKkIo7RgdA
++ mktemp
+ local LAST_ERR=/tmp/tmp.4dKrHQbtIh
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl wait --for=delete namespace demand-backup-physical-aws-13657
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.uKkIo7RgdA
+ cat /tmp/tmp.4dKrHQbtIh
+ rm /tmp/tmp.uKkIo7RgdA /tmp/tmp.4dKrHQbtIh
+ return 0
+ desc 'create namespace demand-backup-physical-aws-13657'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-physical-aws-13657
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-physical-aws-13657
++ mktemp
+ local LAST_OUT=/tmp/tmp.MLfhsmIQhd
++ mktemp
+ local LAST_ERR=/tmp/tmp.JExSFkgobr
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl create namespace demand-backup-physical-aws-13657
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.MLfhsmIQhd
namespace/demand-backup-physical-aws-13657 created
+ cat /tmp/tmp.JExSFkgobr
+ rm /tmp/tmp.MLfhsmIQhd /tmp/tmp.JExSFkgobr
+ return 0
+ set_kube_ctx demand-backup-physical-aws-13657
+ local namespace=demand-backup-physical-aws-13657
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pl82diAZMB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eG3l2xvG0s
++ local exit_status=0
++ local timeout=4
+++ seq 0 2
++ for i in $(seq 0 2)
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 -a -n 1 ']'
++ break
++ cat /tmp/tmp.pl82diAZMB
++ cat /tmp/tmp.eG3l2xvG0s
++ rm /tmp/tmp.pl82diAZMB /tmp/tmp.eG3l2xvG0s
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1 --namespace=demand-backup-physical-aws-13657
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vjx6KV9sWR
++ mktemp
+ local LAST_ERR=/tmp/tmp.UbOsTRCwGV
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1 --namespace=demand-backup-physical-aws-13657
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.Vjx6KV9sWR
Context "gke_cloud-dev-112233_us-central1-a_jen-psmdb-2269-20e73be3-5-cluster1" modified.
+ cat /tmp/tmp.UbOsTRCwGV
+ rm /tmp/tmp.Vjx6KV9sWR /tmp/tmp.UbOsTRCwGV
+ return 0
+ apply_s3_storage_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.07uRsXA5pV
++ mktemp
+ local LAST_ERR=/tmp/tmp.F06QKF2tDD
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.07uRsXA5pV
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
secret/gcp-cs-sa-key-secret created
+ cat /tmp/tmp.F06QKF2tDD
+ rm /tmp/tmp.07uRsXA5pV /tmp/tmp.F06QKF2tDD
+ return 0
+ desc 'Testing on not sharded cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Testing on not sharded cluster
-----------------------------------------------------------------------------------

+ echo 'Creating PSMDB cluster'
Creating PSMDB cluster
+ cluster=some-name
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9YVfos3O01
++ mktemp
+ local LAST_ERR=/tmp/tmp.M3EFayK62L
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.9YVfos3O01
secret/some-users created
+ cat /tmp/tmp.M3EFayK62L
+ rm /tmp/tmp.9YVfos3O01 /tmp/tmp.M3EFayK62L
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/some-name.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/some-name.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/demand-backup-physical-aws/conf/some-name.yml
+ yq eval '(.spec | select(.image == null)).image = "docker.io/perconalab/percona-server-mongodb-operator:main-mongod8.0"'
+ yq eval '(.spec | select(has("pmm"))).pmm.image = "docker.io/percona/pmm-client:2.44.1-1"'
+ yq eval '(.spec | select(has("backup"))).backup.image = "docker.io/perconalab/percona-server-mongodb-operator:main-backup"'
++ mktemp
+ yq eval '.spec.upgradeOptions.apply="Never"'
+ yq eval '(.spec | select(has("initImage"))).initImage = "docker.io/perconalab/percona-server-mongodb-operator:PR-2269-20e73be3"'
+ /usr/sbin/sed -e s/NAME_SPACE/demand-backup-physical-aws-13657/g
+ local LAST_OUT=/tmp/tmp.JbNSxXADoP
++ mktemp
+ local LAST_ERR=/tmp/tmp.4eeET9QaUq
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.JbNSxXADoP
perconaservermongodb.psmdb.percona.com/some-name created
+ cat /tmp/tmp.4eeET9QaUq
+ rm /tmp/tmp.JbNSxXADoP /tmp/tmp.4eeET9QaUq
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/client_with_tls.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SJnyujl3Yd
++ mktemp
+ local LAST_ERR=/tmp/tmp.R0DNm54uN5
+ local exit_status=0
+ local timeout=4
++ seq 0 2
+ for i in $(seq 0 2)
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-psmdb-operator_PR-2269/e2e-tests/conf/client_with_tls.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 -a -n 1 ']'
+ break
+ cat /tmp/tmp.SJnyujl3Yd
deployment.apps/psmdb-client created
+ cat /tmp/tmp.R0DNm54uN5
+ rm /tmp/tmp.SJnyujl3Yd /tmp/tmp.R0DNm54uN5
+ return 0
+ echo 'Check if all pods started'
Check if all pods started
+ wait_for_running some-name-rs0 3
+ local name=some-name-rs0
+ let last_pod=2
+ local check_cluster_readyness=true
+ set_debug
+ [[ 1 == 1 ]]
+ set -o xtrace
+ local rs_name=rs0
+ local cluster_name=some-name
++ seq 0 2
+ for i in $(seq 0 $last_pod)
+ [[ 0 -eq 2 ]]
+ wait_pod some-name-rs0-0
+ local pod=some-name-rs0-0
+ set +o xtrace
waiting for pod/some-name-rs0-0 to be ready....................................................................................................................................................................................................................................................................................................................................Terminated                 kubectl_bin get pod/$pod -o jsonpath='{.status.containerStatuses[0].ready}' 2> /dev/null | grep --quiet 'true'