Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/logs/demand-backup-flow-control-8-0.log
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.35) and server (1.32) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ create_infra demand-backup-flow-control-17600
+ local ns=demand-backup-flow-control-17600
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n demand-backup-flow-control-2099 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.1Li4GKbZcP
++ mktemp
+ local LAST_ERR=/tmp/tmp.4TUmK2E9wj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1Li4GKbZcP
perconaxtradbcluster.pxc.percona.com "some-name" deleted from demand-backup-flow-control-2099 namespace
+ cat /tmp/tmp.4TUmK2E9wj
+ rm /tmp/tmp.1Li4GKbZcP /tmp/tmp.4TUmK2E9wj
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.xt2KDRkHCh
++ mktemp
+ local LAST_ERR=/tmp/tmp.qMdYtDGSnw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xt2KDRkHCh
perconaxtradbclusterbackup.pxc.percona.com "backup1" deleted from demand-backup-flow-control-2099 namespace
perconaxtradbclusterbackup.pxc.percona.com "backup2" deleted from demand-backup-flow-control-2099 namespace
+ cat /tmp/tmp.qMdYtDGSnw
+ rm /tmp/tmp.xt2KDRkHCh /tmp/tmp.qMdYtDGSnw
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rtj88fOd0g
++ mktemp
+ local LAST_ERR=/tmp/tmp.fMZobpThkK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Rtj88fOd0g
No resources found
+ cat /tmp/tmp.fMZobpThkK
+ rm /tmp/tmp.Rtj88fOd0g /tmp/tmp.fMZobpThkK
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ awk '{print$1}'
+ kubectl_bin get ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.E9ZAVqFeKk
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.TYw758p6Aq
+ local LAST_ERR=/tmp/tmp.QUQJHvUh6E
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.advvKnKBxF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.E9ZAVqFeKk
+ cat /tmp/tmp.QUQJHvUh6E
+ rm /tmp/tmp.E9ZAVqFeKk /tmp/tmp.QUQJHvUh6E
+ return 0
namespace "demand-backup-flow-control-2099" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TYw758p6Aq
namespace "pxc-operator" deleted
+ cat /tmp/tmp.advvKnKBxF
+ rm /tmp/tmp.TYw758p6Aq /tmp/tmp.advvKnKBxF
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.WCAacCuhjK
++ mktemp
+ local LAST_ERR=/tmp/tmp.UKbqgF1jF4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WCAacCuhjK
namespace/pxc-operator created
+ cat /tmp/tmp.UKbqgF1jF4
+ rm /tmp/tmp.WCAacCuhjK /tmp/tmp.UKbqgF1jF4
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AFpMls0Thz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9kOSR3h54W
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AFpMls0Thz
++ cat /tmp/tmp.9kOSR3h54W
++ rm /tmp/tmp.AFpMls0Thz /tmp/tmp.9kOSR3h54W
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.MfKPktzk8K
++ mktemp
+ local LAST_ERR=/tmp/tmp.7i5ImCvPJy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MfKPktzk8K
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4" modified.
+ cat /tmp/tmp.7i5ImCvPJy
+ rm /tmp/tmp.MfKPktzk8K /tmp/tmp.7i5ImCvPJy
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yFTbcQPky3
++ mktemp
+ local LAST_ERR=/tmp/tmp.kbM33GfqFA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.yFTbcQPky3
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.kbM33GfqFA
+ rm /tmp/tmp.yFTbcQPky3 /tmp/tmp.kbM33GfqFA
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SKVpWSvWmL
++ mktemp
+ local LAST_ERR=/tmp/tmp.triPOED5ho
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SKVpWSvWmL
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.triPOED5ho
+ rm /tmp/tmp.SKVpWSvWmL /tmp/tmp.triPOED5ho
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.qbjNgWrpu6
++ mktemp
+ local LAST_ERR=/tmp/tmp.QFByFArdIZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.qbjNgWrpu6
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.QFByFArdIZ
+ rm /tmp/tmp.qbjNgWrpu6 /tmp/tmp.QFByFArdIZ
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.kRvSNuyPqm
++ mktemp
+ local LAST_ERR=/tmp/tmp.yZMLqoDz2S
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kRvSNuyPqm
pod/percona-xtradb-cluster-operator-7d5d85b46b-6l64w condition met
+ cat /tmp/tmp.yZMLqoDz2S
+ rm /tmp/tmp.kRvSNuyPqm /tmp/tmp.yZMLqoDz2S
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ head -1
+++ mktemp
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ local LAST_OUT=/tmp/tmp.clk1NoP22F
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VotrIaUTzP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.clk1NoP22F
++ cat /tmp/tmp.VotrIaUTzP
++ rm /tmp/tmp.clk1NoP22F /tmp/tmp.VotrIaUTzP
++ return 0
+ wait_pod percona-xtradb-cluster-operator-7d5d85b46b-6l64w 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-7d5d85b46b-6l64w
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-7d5d85b46b-6l64w
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-7d5d85b46b-6l64w condition met
waiting for pod/percona-xtradb-cluster-operator-7d5d85b46b-6l64w to become Ready.Ok
+ sleep 3
+ create_namespace demand-backup-flow-control-17600
+ local namespace=demand-backup-flow-control-17600
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-flow-control-17600'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-flow-control-17600
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-flow-control-17600
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.1PAJ0B4TKO
++ mktemp
+ xargs kubectl delete ns
+ local LAST_ERR=/tmp/tmp.Koq1JryDS9
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.hibCFAZZtT
++ mktemp
+ local LAST_ERR=/tmp/tmp.l0bPTWq5n5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-flow-control-17600
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1PAJ0B4TKO
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-flow-control-17600
+ cat /tmp/tmp.Koq1JryDS9
+ rm /tmp/tmp.1PAJ0B4TKO /tmp/tmp.Koq1JryDS9
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-flow-control-17600
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.hibCFAZZtT
+ cat /tmp/tmp.l0bPTWq5n5
Error from server (NotFound): namespaces "demand-backup-flow-control-17600" not found
+ rm /tmp/tmp.hibCFAZZtT /tmp/tmp.l0bPTWq5n5
+ return 1
+ :
+ wait_for_delete namespace/demand-backup-flow-control-17600
+ local res=namespace/demand-backup-flow-control-17600
+ echo -n 'waiting for namespace/demand-backup-flow-control-17600 to be deleted'
waiting for namespace/demand-backup-flow-control-17600 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "demand-backup-flow-control-17600" not found
+ desc 'create namespace demand-backup-flow-control-17600'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-flow-control-17600
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-flow-control-17600
++ mktemp
+ local LAST_OUT=/tmp/tmp.5bM5abRVlL
++ mktemp
+ local LAST_ERR=/tmp/tmp.Nv5WgUl4ZR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-flow-control-17600
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5bM5abRVlL
namespace/demand-backup-flow-control-17600 created
+ cat /tmp/tmp.Nv5WgUl4ZR
+ rm /tmp/tmp.5bM5abRVlL /tmp/tmp.Nv5WgUl4ZR
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Kt61s0aM7c
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u9Q2bVJuta
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Kt61s0aM7c
++ cat /tmp/tmp.u9Q2bVJuta
++ rm /tmp/tmp.Kt61s0aM7c /tmp/tmp.u9Q2bVJuta
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4 --namespace=demand-backup-flow-control-17600
++ mktemp
+ local LAST_OUT=/tmp/tmp.LEVfOCSu1Z
++ mktemp
+ local LAST_ERR=/tmp/tmp.22ZPNo73PI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4 --namespace=demand-backup-flow-control-17600
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LEVfOCSu1Z
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2433-150d0617-1-cluster4" modified.
+ cat /tmp/tmp.22ZPNo73PI
+ rm /tmp/tmp.LEVfOCSu1Z /tmp/tmp.22ZPNo73PI
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.gyIVlFhuzQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.PccDxOyJRE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gyIVlFhuzQ
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.PccDxOyJRE
+ rm /tmp/tmp.gyIVlFhuzQ /tmp/tmp.PccDxOyJRE
+ return 0
+ start_minio
+ deploy_helm demand-backup-flow-control-17600
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n '' ]]
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Apr  9 02:29:12 2026
NAMESPACE: demand-backup-flow-control-17600
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-flow-control-17600.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-flow-control-17600 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-flow-control-17600

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-flow-control-17600 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-flow-control-17600 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AP5AsJ08X5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TxA0NQ5ITV
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AP5AsJ08X5
++ cat /tmp/tmp.TxA0NQ5ITV
++ rm /tmp/tmp.AP5AsJ08X5 /tmp/tmp.TxA0NQ5ITV
++ return 0
+ MINIO_POD=minio-service-5fd5489bdc-9t886
+ wait_pod minio-service-5fd5489bdc-9t886
+ local pod=minio-service-5fd5489bdc-9t886
+ local max_retry=480
+ local ns=
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo minio-service-5fd5489bdc-9t886
+ local container=
+ set +o xtrace
pod/minio-service-5fd5489bdc-9t886 condition met
waiting for pod/minio-service-5fd5489bdc-9t886 to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.6n08uTXySQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.qREXfYQ5ek
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6n08uTXySQ
make_bucket: operator-testing
pod "aws-cli" deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.qREXfYQ5ek
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_demand-backup-flow-control-17600
+ rm /tmp/tmp.6n08uTXySQ /tmp/tmp.qREXfYQ5ek
+ return 0
+ log 'creating PXC client'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:30:23+0000]' creating PXC client
[2026-04-09T02:30:23+0000] creating PXC client
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.0GuLNKWnht
++ mktemp
+ local LAST_ERR=/tmp/tmp.PdkYuVL5Wo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/client.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0GuLNKWnht
deployment.apps/pxc-client created
+ cat /tmp/tmp.PdkYuVL5Wo
+ rm /tmp/tmp.0GuLNKWnht /tmp/tmp.PdkYuVL5Wo
+ return 0
+ log 'creating cluster secrets'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:30:25+0000]' creating cluster secrets
[2026-04-09T02:30:25+0000] creating cluster secrets
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.KxJdkIxh6Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.IEiuhBwBhX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KxJdkIxh6Y
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.IEiuhBwBhX
+ rm /tmp/tmp.KxJdkIxh6Y /tmp/tmp.IEiuhBwBhX
+ return 0
+ log 'create PXC cluster: some-name'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:30:28+0000]' create PXC cluster: some-name
[2026-04-09T02:30:28+0000] create PXC cluster: some-name
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/cr.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/cr.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/cr.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/cr.yml
+ local pvc_name=
++ mktemp
+ local LAST_OUT=/tmp/tmp.FhZ0U1kp6R
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/cr.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2433-150d0617#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-flow-control-17600~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
++ mktemp
+ local LAST_ERR=/tmp/tmp.9c9HKbhX49
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.FhZ0U1kp6R
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.9c9HKbhX49
+ rm /tmp/tmp.FhZ0U1kp6R /tmp/tmp.9c9HKbhX49
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/some-name to be ready'
waiting for pxc/some-name to be ready++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u3fjLa1JtD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0cT227hJNs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.u3fjLa1JtD
++ cat /tmp/tmp.0cT227hJNs
++ rm /tmp/tmp.u3fjLa1JtD /tmp/tmp.0cT227hJNs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.crclvT0UbJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K2rUMT9wH0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.crclvT0UbJ
++ cat /tmp/tmp.K2rUMT9wH0
++ rm /tmp/tmp.crclvT0UbJ /tmp/tmp.K2rUMT9wH0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kjuMfeXVOg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2T3hN2UjJU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kjuMfeXVOg
++ cat /tmp/tmp.2T3hN2UjJU
++ rm /tmp/tmp.kjuMfeXVOg /tmp/tmp.2T3hN2UjJU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RMxPQPVbvK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.p7nmVBc50e
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RMxPQPVbvK
++ cat /tmp/tmp.p7nmVBc50e
++ rm /tmp/tmp.RMxPQPVbvK /tmp/tmp.p7nmVBc50e
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.felzc63mxF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bJSuc1HOHU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.felzc63mxF
++ cat /tmp/tmp.bJSuc1HOHU
++ rm /tmp/tmp.felzc63mxF /tmp/tmp.bJSuc1HOHU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BnmOt5q0AX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zAF36FJpZw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BnmOt5q0AX
++ cat /tmp/tmp.zAF36FJpZw
++ rm /tmp/tmp.BnmOt5q0AX /tmp/tmp.zAF36FJpZw
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vdkt3iwZRo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.st30hsHNGj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Vdkt3iwZRo
++ cat /tmp/tmp.st30hsHNGj
++ rm /tmp/tmp.Vdkt3iwZRo /tmp/tmp.st30hsHNGj
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sfGN69tpYI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6nwdEjlg6B
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sfGN69tpYI
++ cat /tmp/tmp.6nwdEjlg6B
++ rm /tmp/tmp.sfGN69tpYI /tmp/tmp.6nwdEjlg6B
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 7 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ANXvc10kK9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KJG2JDGUkn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ANXvc10kK9
++ cat /tmp/tmp.KJG2JDGUkn
++ rm /tmp/tmp.ANXvc10kK9 /tmp/tmp.KJG2JDGUkn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 8 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AhzfMjddJQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6gMIw12Nsv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AhzfMjddJQ
++ cat /tmp/tmp.6gMIw12Nsv
++ rm /tmp/tmp.AhzfMjddJQ /tmp/tmp.6gMIw12Nsv
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 9 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FGhbBGBf5f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eBa5RArmzF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FGhbBGBf5f
++ cat /tmp/tmp.eBa5RArmzF
++ rm /tmp/tmp.FGhbBGBf5f /tmp/tmp.eBa5RArmzF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 10 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XgDTD7MFhF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J2SRtgZMSt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.XgDTD7MFhF
++ cat /tmp/tmp.J2SRtgZMSt
++ rm /tmp/tmp.XgDTD7MFhF /tmp/tmp.J2SRtgZMSt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 11 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.o6O6qqDSOz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QroV5lLzmm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.o6O6qqDSOz
++ cat /tmp/tmp.QroV5lLzmm
++ rm /tmp/tmp.o6O6qqDSOz /tmp/tmp.QroV5lLzmm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 12 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cmFsJVGvt3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WcAjSGFdiS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cmFsJVGvt3
++ cat /tmp/tmp.WcAjSGFdiS
++ rm /tmp/tmp.cmFsJVGvt3 /tmp/tmp.WcAjSGFdiS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 13 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZYPCJJJD60
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KWbP775DPJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ZYPCJJJD60
++ cat /tmp/tmp.KWbP775DPJ
++ rm /tmp/tmp.ZYPCJJJD60 /tmp/tmp.KWbP775DPJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 14 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yf0ZVzu3nt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.U0FAu795Vy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yf0ZVzu3nt
++ cat /tmp/tmp.U0FAu795Vy
++ rm /tmp/tmp.yf0ZVzu3nt /tmp/tmp.U0FAu795Vy
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 15 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YxDO7ctrWW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2CzHtqBbpv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YxDO7ctrWW
++ cat /tmp/tmp.2CzHtqBbpv
++ rm /tmp/tmp.YxDO7ctrWW /tmp/tmp.2CzHtqBbpv
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 16 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DLNSi9SSCm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mVQnJSqAht
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DLNSi9SSCm
++ cat /tmp/tmp.mVQnJSqAht
++ rm /tmp/tmp.DLNSi9SSCm /tmp/tmp.mVQnJSqAht
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 17 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xY392wFQQw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.S0iVlZJXsr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xY392wFQQw
++ cat /tmp/tmp.S0iVlZJXsr
++ rm /tmp/tmp.xY392wFQQw /tmp/tmp.S0iVlZJXsr
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 18 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RP9Dr2Sbbu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IwE03ufcw5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RP9Dr2Sbbu
++ cat /tmp/tmp.IwE03ufcw5
++ rm /tmp/tmp.RP9Dr2Sbbu /tmp/tmp.IwE03ufcw5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 19 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.60BHcG3DV9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YyUocrn3GT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.60BHcG3DV9
++ cat /tmp/tmp.YyUocrn3GT
++ rm /tmp/tmp.60BHcG3DV9 /tmp/tmp.YyUocrn3GT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 20 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7sChzDKeCF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dd9K6mg7Qf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7sChzDKeCF
++ cat /tmp/tmp.Dd9K6mg7Qf
++ rm /tmp/tmp.7sChzDKeCF /tmp/tmp.Dd9K6mg7Qf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 21 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gqEdciZToH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tijnhwOn3f
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gqEdciZToH
++ cat /tmp/tmp.tijnhwOn3f
++ rm /tmp/tmp.gqEdciZToH /tmp/tmp.tijnhwOn3f
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 22 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5eLioFPz79
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uGODCdBqh5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5eLioFPz79
++ cat /tmp/tmp.uGODCdBqh5
++ rm /tmp/tmp.5eLioFPz79 /tmp/tmp.uGODCdBqh5
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 23 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dxqnYCURzC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ieWBJ9EH3i
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dxqnYCURzC
++ cat /tmp/tmp.ieWBJ9EH3i
++ rm /tmp/tmp.dxqnYCURzC /tmp/tmp.ieWBJ9EH3i
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 24 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BMTNdYXbGa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.u0kGvkxSPh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BMTNdYXbGa
++ cat /tmp/tmp.u0kGvkxSPh
++ rm /tmp/tmp.BMTNdYXbGa /tmp/tmp.u0kGvkxSPh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 25 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eweuEXqA95
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q2b5stGCkU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eweuEXqA95
++ cat /tmp/tmp.Q2b5stGCkU
++ rm /tmp/tmp.eweuEXqA95 /tmp/tmp.Q2b5stGCkU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 26 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ixf7dw3sUn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vk2rVJi9H7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ixf7dw3sUn
++ cat /tmp/tmp.Vk2rVJi9H7
++ rm /tmp/tmp.Ixf7dw3sUn /tmp/tmp.Vk2rVJi9H7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 27 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Cy2krkN0is
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IO4mwjjCeZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Cy2krkN0is
++ cat /tmp/tmp.IO4mwjjCeZ
++ rm /tmp/tmp.Cy2krkN0is /tmp/tmp.IO4mwjjCeZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 28 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SgM8jibdJz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9uyaC3t6EN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SgM8jibdJz
++ cat /tmp/tmp.9uyaC3t6EN
++ rm /tmp/tmp.SgM8jibdJz /tmp/tmp.9uyaC3t6EN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 29 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.exQMoQMpWL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jhD9U222Gb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.exQMoQMpWL
++ cat /tmp/tmp.jhD9U222Gb
++ rm /tmp/tmp.exQMoQMpWL /tmp/tmp.jhD9U222Gb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 30 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lmUEdboeBW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8in5kzEkNt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lmUEdboeBW
++ cat /tmp/tmp.8in5kzEkNt
++ rm /tmp/tmp.lmUEdboeBW /tmp/tmp.8in5kzEkNt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 31 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.54POvCoUZ1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dA0nS3Ed3l
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.54POvCoUZ1
++ cat /tmp/tmp.dA0nS3Ed3l
++ rm /tmp/tmp.54POvCoUZ1 /tmp/tmp.dA0nS3Ed3l
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 32 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lM3JCs4Us3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KjKecjhNtc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lM3JCs4Us3
++ cat /tmp/tmp.KjKecjhNtc
++ rm /tmp/tmp.lM3JCs4Us3 /tmp/tmp.KjKecjhNtc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 33 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CmA1FpNqqA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XfnB9UC2qX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.CmA1FpNqqA
++ cat /tmp/tmp.XfnB9UC2qX
++ rm /tmp/tmp.CmA1FpNqqA /tmp/tmp.XfnB9UC2qX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 34 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wGt9Nz0abv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.06VVyne4M9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wGt9Nz0abv
++ cat /tmp/tmp.06VVyne4M9
++ rm /tmp/tmp.wGt9Nz0abv /tmp/tmp.06VVyne4M9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 35 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.82vt5kakRH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q4rcQiUkuF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.82vt5kakRH
++ cat /tmp/tmp.q4rcQiUkuF
++ rm /tmp/tmp.82vt5kakRH /tmp/tmp.q4rcQiUkuF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 36 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6v4QX7bCNz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ivwaN82Dwr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6v4QX7bCNz
++ cat /tmp/tmp.ivwaN82Dwr
++ rm /tmp/tmp.6v4QX7bCNz /tmp/tmp.ivwaN82Dwr
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.M2UkLOM5xf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SyxWz82H9n
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.M2UkLOM5xf
++ cat /tmp/tmp.SyxWz82H9n
++ rm /tmp/tmp.M2UkLOM5xf /tmp/tmp.SyxWz82H9n
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.5CAhYNHIbs
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.MF3hdVjpJg
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.5CAhYNHIbs
+++++ cat /tmp/tmp.MF3hdVjpJg
+++++ rm /tmp/tmp.5CAhYNHIbs /tmp/tmp.MF3hdVjpJg
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.94hltD6cdO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.K8nnObRH2p
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.94hltD6cdO
++ cat /tmp/tmp.K8nnObRH2p
++ rm /tmp/tmp.94hltD6cdO /tmp/tmp.K8nnObRH2p
++ return 0
+ [[ 2 == \2 ]]
+ echo

+ desc 'CASE 1: startingDeadlineSeconds'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 1: startingDeadlineSeconds
-----------------------------------------------------------------------------------

+ log 'setting startingDeadlineSeconds to 20'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:34:41+0000]' setting startingDeadlineSeconds to 20
[2026-04-09T02:34:41+0000] setting startingDeadlineSeconds to 20
+ kubectl_bin patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 20}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.jOCFEy3OW2
++ mktemp
+ local LAST_ERR=/tmp/tmp.vadpXz6lyw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 20}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jOCFEy3OW2
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.vadpXz6lyw
+ rm /tmp/tmp.jOCFEy3OW2 /tmp/tmp.vadpXz6lyw
+ return 0
+ log 'create dummy lock to block backup'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:34:42+0000]' create dummy lock to block backup
[2026-04-09T02:34:42+0000] create dummy lock to block backup
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/lease.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Yxim3v5En6
++ mktemp
+ local LAST_ERR=/tmp/tmp.0RKJ5YwQut
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/lease.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Yxim3v5En6
lease.coordination.k8s.io/pxc-some-name-backup-lock created
+ cat /tmp/tmp.0RKJ5YwQut
+ rm /tmp/tmp.Yxim3v5En6 /tmp/tmp.0RKJ5YwQut
+ return 0
+ log 'creating pxc-backup/backup1'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:34:43+0000]' creating pxc-backup/backup1
[2026-04-09T02:34:43+0000] creating pxc-backup/backup1
+ run_backup backup1
+ local name=backup1
+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup1"' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5SiIgchPcj
++ mktemp
+ local LAST_ERR=/tmp/tmp.9FYpBR3zWT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5SiIgchPcj
perconaxtradbclusterbackup.pxc.percona.com/backup1 created
+ cat /tmp/tmp.9FYpBR3zWT
+ rm /tmp/tmp.5SiIgchPcj /tmp/tmp.9FYpBR3zWT
+ return 0
+ wait_backup backup1 Failed
+ local backup=backup1
+ local status=Failed
+ set +o xtrace
waiting for pxc-backup/backup1 to reach Failed state............Failed
+ log 'operator should fail backup'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:35:07+0000]' operator should fail backup
[2026-04-09T02:35:07+0000] operator should fail backup
+ check_backup_error backup1 'starting deadline seconds exceeded'
+ local name=backup1
+ local 'expected=starting deadline seconds exceeded'
+ yq .status.error
+ grep 'starting deadline seconds exceeded'
+ kubectl_bin get pxc-backup backup1 -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.irp607IoPH
++ mktemp
+ local LAST_ERR=/tmp/tmp.GvghgmFtPY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get pxc-backup backup1 -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.irp607IoPH
+ cat /tmp/tmp.GvghgmFtPY
+ rm /tmp/tmp.irp607IoPH /tmp/tmp.GvghgmFtPY
+ return 0
starting deadline seconds exceeded: deadline exceeded
+ log 'operator successfully failed the backup job'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:35:08+0000]' operator successfully failed the backup job
[2026-04-09T02:35:08+0000] operator successfully failed the backup job
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/lease.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.BAbH4vvQJ0
++ mktemp
+ local LAST_ERR=/tmp/tmp.ef6Jc5uz7g
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/lease.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BAbH4vvQJ0
lease.coordination.k8s.io "pxc-some-name-backup-lock" deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.ef6Jc5uz7g
+ rm /tmp/tmp.BAbH4vvQJ0 /tmp/tmp.ef6Jc5uz7g
+ return 0
+ desc 'CASE 1: startingDeadlineSeconds PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 1: startingDeadlineSeconds PASSED
-----------------------------------------------------------------------------------

+ desc 'CASE 2: suspend and resume'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 2: suspend and resume
-----------------------------------------------------------------------------------

+ log 'setting suspendedDeadlineSeconds to 300'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:35:10+0000]' setting suspendedDeadlineSeconds to 300
[2026-04-09T02:35:10+0000] setting suspendedDeadlineSeconds to 300
+ kubectl_bin patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 1800, "suspendedDeadlineSeconds": 300}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.sQwzs3uxkP
++ mktemp
+ local LAST_ERR=/tmp/tmp.b0jQq7XA0J
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 1800, "suspendedDeadlineSeconds": 300}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.sQwzs3uxkP
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.b0jQq7XA0J
+ rm /tmp/tmp.sQwzs3uxkP /tmp/tmp.b0jQq7XA0J
+ return 0
+ log 'creating pxc-backup/backup2'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:35:11+0000]' creating pxc-backup/backup2
[2026-04-09T02:35:11+0000] creating pxc-backup/backup2
+ run_backup backup2
+ local name=backup2
+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup2"' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.VM0M7DE7pK
++ mktemp
+ local LAST_ERR=/tmp/tmp.CvPs5R4pSo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VM0M7DE7pK
perconaxtradbclusterbackup.pxc.percona.com/backup2 created
+ cat /tmp/tmp.CvPs5R4pSo
+ rm /tmp/tmp.VM0M7DE7pK /tmp/tmp.CvPs5R4pSo
+ return 0
+ wait_backup backup2 Running
+ local backup=backup2
+ local status=Running
+ set +o xtrace
waiting for pxc-backup/backup2 to reach Running state......Running
+ log 'deleting pod/some-name-pxc-2'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:35:26+0000]' deleting pod/some-name-pxc-2
[2026-04-09T02:35:26+0000] deleting pod/some-name-pxc-2
+ kubectl_bin delete pod some-name-pxc-2 --force
++ mktemp
+ local LAST_OUT=/tmp/tmp.vV61VebTDp
++ mktemp
+ local LAST_ERR=/tmp/tmp.fyiWtCnqiD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod some-name-pxc-2 --force
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vV61VebTDp
pod "some-name-pxc-2" force deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.fyiWtCnqiD
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.vV61VebTDp /tmp/tmp.fyiWtCnqiD
+ return 0
+ wait_backup backup2 Suspended
+ local backup=backup2
+ local status=Suspended
+ set +o xtrace
waiting for pxc-backup/backup2 to reach Suspended state.Suspended
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/some-name to be ready'
waiting for pxc/some-name to be ready++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.I8C1noMa4U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ocrv5npGAm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.I8C1noMa4U
++ cat /tmp/tmp.ocrv5npGAm
++ rm /tmp/tmp.I8C1noMa4U /tmp/tmp.ocrv5npGAm
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BNQ47ni6rt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.uJcQHrY3tH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BNQ47ni6rt
++ cat /tmp/tmp.uJcQHrY3tH
++ rm /tmp/tmp.BNQ47ni6rt /tmp/tmp.uJcQHrY3tH
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aQUhDX5bzx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xBepuBGZan
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aQUhDX5bzx
++ cat /tmp/tmp.xBepuBGZan
++ rm /tmp/tmp.aQUhDX5bzx /tmp/tmp.xBepuBGZan
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KB8fb2lGhu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8AQ3aPVoIU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KB8fb2lGhu
++ cat /tmp/tmp.8AQ3aPVoIU
++ rm /tmp/tmp.KB8fb2lGhu /tmp/tmp.8AQ3aPVoIU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2Stm6wTn9i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TcgljhuySk
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2Stm6wTn9i
++ cat /tmp/tmp.TcgljhuySk
++ rm /tmp/tmp.2Stm6wTn9i /tmp/tmp.TcgljhuySk
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FLJr4Mm9Ys
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Qhl5365VBF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FLJr4Mm9Ys
++ cat /tmp/tmp.Qhl5365VBF
++ rm /tmp/tmp.FLJr4Mm9Ys /tmp/tmp.Qhl5365VBF
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.BllAFpeZdC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1s7utVbxdt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.BllAFpeZdC
++ cat /tmp/tmp.1s7utVbxdt
++ rm /tmp/tmp.BllAFpeZdC /tmp/tmp.1s7utVbxdt
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.glo0hwi6rA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FggWkk1KEo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.glo0hwi6rA
++ cat /tmp/tmp.FggWkk1KEo
++ rm /tmp/tmp.glo0hwi6rA /tmp/tmp.FggWkk1KEo
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pMOGDb3EPR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X9FecnaJI4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.pMOGDb3EPR
++ cat /tmp/tmp.X9FecnaJI4
++ rm /tmp/tmp.pMOGDb3EPR /tmp/tmp.X9FecnaJI4
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.NdrZJrLEMk
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.QgjYjn6hzG
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.NdrZJrLEMk
+++++ cat /tmp/tmp.QgjYjn6hzG
+++++ rm /tmp/tmp.NdrZJrLEMk /tmp/tmp.QgjYjn6hzG
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fVA6RP27RH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iNG5qZ54V1
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fVA6RP27RH
++ cat /tmp/tmp.iNG5qZ54V1
++ rm /tmp/tmp.fVA6RP27RH /tmp/tmp.iNG5qZ54V1
++ return 0
+ [[ 2 == \2 ]]
+ echo

+ wait_backup backup2 Succeeded
+ local backup=backup2
+ local status=Succeeded
+ set +o xtrace
waiting for pxc-backup/backup2 to reach Succeeded state.....................................Succeeded
+ desc 'CASE 2: suspend and resume PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 2: suspend and resume PASSED
-----------------------------------------------------------------------------------

+ desc 'CASE 3: suspendedDeadlineSeconds'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 3: suspendedDeadlineSeconds
-----------------------------------------------------------------------------------

+ log 'setting suspendedDeadlineSeconds to 10'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:37:31+0000]' setting suspendedDeadlineSeconds to 10
[2026-04-09T02:37:31+0000] setting suspendedDeadlineSeconds to 10
+ kubectl_bin patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 1800, "suspendedDeadlineSeconds": 10}}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0ZdOxNXE5u
++ mktemp
+ local LAST_ERR=/tmp/tmp.8LBSag53yN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge -p '{"spec": {"backup": {"startingDeadlineSeconds": 1800, "suspendedDeadlineSeconds": 10}}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0ZdOxNXE5u
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.8LBSag53yN
+ rm /tmp/tmp.0ZdOxNXE5u /tmp/tmp.8LBSag53yN
+ return 0
+ log 'creating pxc-backup/backup3'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:37:33+0000]' creating pxc-backup/backup3
[2026-04-09T02:37:33+0000] creating pxc-backup/backup3
+ run_backup backup3
+ local name=backup3
+ kubectl_bin apply -f -
+ yq eval '.metadata.name = "backup3"' /mnt/jenkins/workspace/cloud-pxc-operator_PR-2433/e2e-tests/demand-backup-flow-control/conf/backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.zPwTbEBhzY
++ mktemp
+ local LAST_ERR=/tmp/tmp.SoeKosHbij
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zPwTbEBhzY
perconaxtradbclusterbackup.pxc.percona.com/backup3 created
+ cat /tmp/tmp.SoeKosHbij
+ rm /tmp/tmp.zPwTbEBhzY /tmp/tmp.SoeKosHbij
+ return 0
+ wait_backup backup3 Running
+ local backup=backup3
+ local status=Running
+ set +o xtrace
waiting for pxc-backup/backup3 to reach Running state....Running
+ log 'deleting pod/some-name-pxc-2'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:37:42+0000]' deleting pod/some-name-pxc-2
[2026-04-09T02:37:42+0000] deleting pod/some-name-pxc-2
+ kubectl_bin delete pod some-name-pxc-2 --force
++ mktemp
+ local LAST_OUT=/tmp/tmp.A8a9D4xoL6
++ mktemp
+ local LAST_ERR=/tmp/tmp.yiYm3DgEd2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pod some-name-pxc-2 --force
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.A8a9D4xoL6
pod "some-name-pxc-2" force deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.yiYm3DgEd2
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
+ rm /tmp/tmp.A8a9D4xoL6 /tmp/tmp.yiYm3DgEd2
+ return 0
+ wait_backup backup3 Suspended
+ local backup=backup3
+ local status=Suspended
+ set +o xtrace
waiting for pxc-backup/backup3 to reach Suspended state.Suspended
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=300
+ sleep 7
+ echo -n 'waiting for pxc/some-name to be ready'
waiting for pxc/some-name to be ready++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LFZ7LKjFR4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lljEapUuI4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LFZ7LKjFR4
++ cat /tmp/tmp.lljEapUuI4
++ rm /tmp/tmp.LFZ7LKjFR4 /tmp/tmp.lljEapUuI4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 0 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wjUFcm0eTI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2uP3oUyx2C
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wjUFcm0eTI
++ cat /tmp/tmp.2uP3oUyx2C
++ rm /tmp/tmp.wjUFcm0eTI /tmp/tmp.2uP3oUyx2C
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 1 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YPplp8DeTL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4t1XYxhYnX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YPplp8DeTL
++ cat /tmp/tmp.4t1XYxhYnX
++ rm /tmp/tmp.YPplp8DeTL /tmp/tmp.4t1XYxhYnX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 2 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9tkgbPsV2Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nRQNoApG9Q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9tkgbPsV2Y
++ cat /tmp/tmp.nRQNoApG9Q
++ rm /tmp/tmp.9tkgbPsV2Y /tmp/tmp.nRQNoApG9Q
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 3 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EkjUPnkLc7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mDR25FgfHq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EkjUPnkLc7
++ cat /tmp/tmp.mDR25FgfHq
++ rm /tmp/tmp.EkjUPnkLc7 /tmp/tmp.mDR25FgfHq
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 4 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IXCKvv0Ohc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AqjRficMNc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IXCKvv0Ohc
++ cat /tmp/tmp.AqjRficMNc
++ rm /tmp/tmp.IXCKvv0Ohc /tmp/tmp.AqjRficMNc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 5 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aY3myDyz53
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JwoenA1d75
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aY3myDyz53
++ cat /tmp/tmp.JwoenA1d75
++ rm /tmp/tmp.aY3myDyz53 /tmp/tmp.JwoenA1d75
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo -n .
.+ sleep 5
+ [[ 6 -ge 300 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ek9BzbjiMF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BQD09DPQfN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ek9BzbjiMF
++ cat /tmp/tmp.BQD09DPQfN
++ rm /tmp/tmp.Ek9BzbjiMF /tmp/tmp.BQD09DPQfN
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.5Dc4Pc4OYe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nHJw4A42rY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.5Dc4Pc4OYe
++ cat /tmp/tmp.nHJw4A42rY
++ rm /tmp/tmp.5Dc4Pc4OYe /tmp/tmp.nHJw4A42rY
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.ib7UYS0Blv
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.sO46XcbbUH
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.ib7UYS0Blv
+++++ cat /tmp/tmp.sO46XcbbUH
+++++ rm /tmp/tmp.ib7UYS0Blv /tmp/tmp.sO46XcbbUH
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-haproxy
++++ return
+++ local cluster_proxy=some-name-haproxy
+++ echo haproxy
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.x8piewj6DB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.n2jjdPvDJQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.haproxy.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.x8piewj6DB
++ cat /tmp/tmp.n2jjdPvDJQ
++ rm /tmp/tmp.x8piewj6DB /tmp/tmp.n2jjdPvDJQ
++ return 0
+ [[ 2 == \2 ]]
+ echo

+ wait_backup backup3 Failed
+ local backup=backup3
+ local status=Failed
+ set +o xtrace
waiting for pxc-backup/backup3 to reach Failed stateFailed
+ log 'operator should fail backup'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:38:43+0000]' operator should fail backup
[2026-04-09T02:38:43+0000] operator should fail backup
+ check_backup_error backup3 'suspended deadline seconds exceeded'
+ local name=backup3
+ local 'expected=suspended deadline seconds exceeded'
+ yq .status.error
+ grep 'suspended deadline seconds exceeded'
+ kubectl_bin get pxc-backup backup3 -o yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.aV1xwSe1p7
++ mktemp
+ local LAST_ERR=/tmp/tmp.pxsd7aUDG0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get pxc-backup backup3 -o yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aV1xwSe1p7
+ cat /tmp/tmp.pxsd7aUDG0
+ rm /tmp/tmp.aV1xwSe1p7 /tmp/tmp.pxsd7aUDG0
+ return 0
suspended deadline seconds exceeded: deadline exceeded
+ desc 'CASE 3: suspendedDeadlineSeconds PASSED'
+ set +o xtrace


-----------------------------------------------------------------------------------
CASE 3: suspendedDeadlineSeconds PASSED
-----------------------------------------------------------------------------------

+ log 'test passed'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-04-09T02:38:44+0000]' test passed
[2026-04-09T02:38:44+0000] test passed
+ destroy demand-backup-flow-control-17600
+ local namespace=demand-backup-flow-control-17600
+ local ignore_logs=true
+ [[ 0 == 1 ]]
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
++ get_operator_pod
+ grep -v 'the object has been modified'
++ local label_prefix=app.kubernetes.io/
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v 'get backup status: Job.batch'
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+ tee /tmp/tmp.6GOpIwNc7Z/operator.log
+ sort -u
+ grep -v level=info
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ydB8CdlDYs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EMzvHyUNPI
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ydB8CdlDYs
++ cat /tmp/tmp.EMzvHyUNPI
++ rm /tmp/tmp.ydB8CdlDYs /tmp/tmp.EMzvHyUNPI
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-7d5d85b46b-6l64w
++ mktemp
+ local LAST_OUT=/tmp/tmp.AhP5C8mRu7
++ mktemp
+ local LAST_ERR=/tmp/tmp.6O090bs39n
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-7d5d85b46b-6l64w
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AhP5C8mRu7
+ cat /tmp/tmp.6O090bs39n
+ rm /tmp/tmp.AhP5C8mRu7 /tmp/tmp.6O090bs39n
+ return 0
2026-04-09T02:28:24.747Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.32.13-gke.1205000"}
2026-04-09T02:28:24.748Z	INFO	setup	Feature gates	{"PXCO_FEATURE_GATES": "", "enabled": ""}
2026-04-09T02:28:24.748Z	INFO	setup	Manager starting up	{"gitCommit": "150d06179c61737b08885893a1a3dd8ba7b2b10c", "gitBranch": "PR-2433-150d0617", "buildTime": "2026-04-09T01:11:17Z", "goVersion": "go1.25.9", "os": "linux", "arch": "amd64"}
2026-04-09T02:28:24.751Z	INFO	setup	Registering Components.
2026-04-09T02:28:25.140Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2026-04-09T02:28:25.140Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2026-04-09T02:28:25.140Z	INFO	controller-runtime.metrics	Starting metrics server
2026-04-09T02:28:25.140Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2026-04-09T02:28:25.140Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2026-04-09T02:28:25.140Z	INFO	controller-runtime.webhook	Starting webhook server
2026-04-09T02:28:25.140Z	INFO	setup	Starting the Cmd.
2026-04-09T02:28:25.140Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2026-04-09T02:28:25.141Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2026-04-09T02:28:25.241Z	INFO	Attempting to acquire leader lease...	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-09T02:28:25.271Z	DEBUG	events	percona-xtradb-cluster-operator-7d5d85b46b-6l64w_0223cf61-4294-402d-88c3-ab894276770a became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"b9ec37fb-8c0d-4dd0-9004-5d186035a310","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1775701705265167009"}, "reason": "LeaderElection"}
2026-04-09T02:28:25.271Z	INFO	Successfully acquired lease	{"lock": "pxc-operator/08db1feb.percona.com"}
2026-04-09T02:28:25.272Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2026-04-09T02:28:25.272Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"}
2026-04-09T02:28:25.272Z	INFO	Starting EventSource	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"}
2026-04-09T02:28:25.272Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2026-04-09T02:28:25.373Z	INFO	Starting Controller	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"}
2026-04-09T02:28:25.373Z	INFO	Starting Controller	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"}
2026-04-09T02:28:25.373Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1}
2026-04-09T02:28:25.373Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1}
2026-04-09T02:28:25.474Z	INFO	Starting Controller	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"}
2026-04-09T02:28:25.474Z	INFO	Starting workers	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1}
2026-04-09T02:30:31.276Z	INFO	Set CR version	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "version": "1.20.0"}
2026-04-09T02:30:31.599Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "auto-some-name-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T02:30:31.617Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2026-04-09T02:30:31.734Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-haproxy", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T02:30:31.773Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2026-04-09T02:30:31.824Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T02:30:31.851Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T02:30:31.906Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-haproxy", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T02:30:31.991Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "d277e29b-4d04-403c-aebf-924215737e04", "object": "some-name-haproxy-replicas", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2026-04-09T02:30:32.858Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "505dbda3-8252-45b2-b899-4178bae35a69", "object": "some-name-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T02:30:32.880Z	DEBUG	Creating object	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "505dbda3-8252-45b2-b899-4178bae35a69", "object": "some-name-haproxy", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2026-04-09T02:31:54.453Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c", "user": "operator"}
2026-04-09T02:31:54.486Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c", "user": "monitor"}
2026-04-09T02:31:54.545Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c"}
2026-04-09T02:31:54.585Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c"}
2026-04-09T02:31:54.615Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c", "user": "xtrabackup"}
2026-04-09T02:31:54.669Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c"}
2026-04-09T02:31:54.699Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "a893cf84-4f35-4ec4-b84b-ce5a0daaa87c", "user": "replication"}
2026-04-09T02:34:31.673Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "af98462d-f6f6-4dc0-9e36-61a0f5a463e7", "user": "root"}
2026-04-09T02:34:31.797Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"some-name","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "some-name", "reconcileID": "af98462d-f6f6-4dc0-9e36-61a0f5a463e7", "new version": "8.0.43-34.1"}
2026-04-09T02:34:45.259Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "ab7f3e20-15ff-4280-b793-5bc69a490e34", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:34:45.360Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "02e2ec80-479a-41b5-aaa2-7613116fab6b", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:34:45.360Z	INFO	Another backup is holding the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "02e2ec80-479a-41b5-aaa2-7613116fab6b", "cluster": "some-name", "storage": "minio", "holder": "dummy-backup"}
2026-04-09T02:34:45.360Z	INFO	Another backup is holding the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "ab7f3e20-15ff-4280-b793-5bc69a490e34", "cluster": "some-name", "storage": "minio", "holder": "dummy-backup"}
2026-04-09T02:34:50.361Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "b2bd95fe-a1f7-4bd3-acf0-79ca0b4ba7ad", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:34:50.361Z	INFO	Another backup is holding the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "b2bd95fe-a1f7-4bd3-acf0-79ca0b4ba7ad", "cluster": "some-name", "storage": "minio", "holder": "dummy-backup"}
2026-04-09T02:34:55.362Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "106b8fbc-05dc-4e88-ade7-e37c22170698", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:34:55.362Z	INFO	Another backup is holding the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "106b8fbc-05dc-4e88-ade7-e37c22170698", "cluster": "some-name", "storage": "minio", "holder": "dummy-backup"}
2026-04-09T02:35:00.362Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "87d92ac5-a496-41a3-8533-2eb811ace8ba", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:00.363Z	INFO	Another backup is holding the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "87d92ac5-a496-41a3-8533-2eb811ace8ba", "cluster": "some-name", "storage": "minio", "holder": "dummy-backup"}
2026-04-09T02:35:05.364Z	INFO	Backup didn't start in startingDeadlineSeconds, failing the backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "b1aea4e6-d297-4641-a358-c156cd27a00e", "startingDeadlineSeconds": 20, "passedSeconds": 20.364179863}
2026-04-09T02:35:05.391Z	ERROR	failed to release the lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "b1aea4e6-d297-4641-a358-c156cd27a00e", "cluster": "some-name", "error": "not the holder", "errorVerbose": "not the holder\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/k8s.init\n\t<autogenerated>:1\nruntime.doInit1\n\t/usr/local/go/src/runtime/proc.go:7670\nruntime.doInit\n\t/usr/local/go/src/runtime/proc.go:7637\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:256\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
2026-04-09T02:35:05.391Z	INFO	Releasing backup lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup1","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup1", "reconcileID": "b1aea4e6-d297-4641-a358-c156cd27a00e", "cluster": "some-name", "lease": "pxc-some-name-backup-lock"}
2026-04-09T02:35:15.331Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "0bdde527-1f6e-47d4-9b80-e9b2111fd589", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:15.478Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "06ae5d0f-5782-4366-9e2b-452a043e35a8", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:15.478Z	INFO	Created a new backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "0bdde527-1f6e-47d4-9b80-e9b2111fd589", "namespace": "demand-backup-flow-control-17600", "name": "xb-backup2"}
2026-04-09T02:35:15.569Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "52f3434b-b9b5-4d8d-9eef-495e7ae97601", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:20.668Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "ff51a7cb-22f1-4d61-b6bf-fb2c5368ee90", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:25.714Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "5adb73a4-9728-4dee-aa3c-f38b7e5714a8", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:25.790Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "f5dfa6e8-bda7-4d70-915e-35fe89c6972e", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:35:30.835Z	INFO	Suspending backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "e92c65e9-405f-4f37-a8b0-c8a299407daf", "job": "xb-backup2", "clusterStatus": "initializing", "readyPXC": 2}
2026-04-09T02:35:30.902Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "e92c65e9-405f-4f37-a8b0-c8a299407daf", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:30.911Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "9a686292-f71e-4968-bbb0-4e6c32a64bb5", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:35.911Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "509559a2-eefb-423f-a4f1-6dc4a3029a25", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:40.912Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "5b10394c-5382-43f0-a64a-4964a032a059", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:45.913Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "021ddc8f-016f-4499-8a70-aafba4106182", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:50.914Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "651470a7-8af5-4f8d-893c-3561d12f5542", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:35:55.915Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "055033d6-75bc-4b33-979d-a20e55ec0657", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:00.915Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "14f3ee52-1c41-412e-b039-a201f95b054c", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:05.916Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "12eacc4f-b123-4fb3-af68-b4854cba0343", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:10.917Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "b2f82fb4-3739-47dd-90d7-2010242cd1d2", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:15.918Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "deadc7e5-2c2f-4165-8a4c-155aa5f044d1", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:20.918Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "19c2c5ae-02fd-44f7-a5c0-7ccd41645603", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:36:25.919Z	INFO	Resuming backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "8c34e859-3205-49cd-9fa6-70a70e10a902", "job": "xb-backup2", "clusterStatus": "ready", "readyPXC": 3}
2026-04-09T02:36:25.985Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "8c34e859-3205-49cd-9fa6-70a70e10a902", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:26.028Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "25c0ec7a-82df-4096-9161-3f4600658e0e", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:31.113Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "2fd8a5c3-0bde-40bc-8e43-883f740eab50", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:31.180Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "897f5da2-72ba-4b98-aa72-5969aba4eb8c", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:36.222Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "3c28b3fe-9336-44a2-9c50-d6cca5d9a71f", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:41.260Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "d5ec36d8-4d9f-4ed8-83f0-3ea26036cd3d", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:46.297Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "2b6939ee-8879-4f9c-ae56-0fbd66fb47d4", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:51.343Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "b7ec67a2-6b93-4f3b-9579-8ec63878eacb", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:36:56.379Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "2181742d-bcd3-41f3-ab5b-7d2efb63661f", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:01.420Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "8f69d012-f797-4b8f-9121-744f42403fdd", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:06.457Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "88887c35-4ad0-4889-80da-a87ae8567be0", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:11.493Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "4845c6ca-bcfa-4931-be51-8b011127d4b1", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:16.534Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "3dd9d25d-b09d-4807-b1aa-24bc2c6e4eb6", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:21.576Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "7bfdf829-5be4-4ab2-bf86-c7a9711eaaa1", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:26.614Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "c1fe6773-c0c7-4501-8f6e-80c6c6d42cd6", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:31.654Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "468dcb8c-da58-4059-ace0-3df857491fd9", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:31.717Z	DEBUG	Removing mysql-init secret	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "468dcb8c-da58-4059-ace0-3df857491fd9", "job": "xb-backup2", "secret": "some-name-mysql-init"}
2026-04-09T02:37:31.717Z	INFO	Backup succeeded	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "468dcb8c-da58-4059-ace0-3df857491fd9", "job": "xb-backup2"}
2026-04-09T02:37:31.761Z	INFO	Releasing backup lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup2","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup2", "reconcileID": "468dcb8c-da58-4059-ace0-3df857491fd9", "cluster": "some-name", "storage": "minio", "lease": "pxc-some-name-backup-lock"}
2026-04-09T02:37:35.604Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "8bc3303c-7700-4b9b-a97f-357888d0af18", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:35.653Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "5ce3e790-dd67-4c98-b49f-e798046534c7", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:35.653Z	INFO	Created a new backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "8bc3303c-7700-4b9b-a97f-357888d0af18", "namespace": "demand-backup-flow-control-17600", "name": "xb-backup3"}
2026-04-09T02:37:35.730Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "047eb683-0e44-44f4-9dc8-7054ec5f9d4a", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:40.880Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "ace49cc4-1b5a-4dc6-8204-8ab6cbfd73fa", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:40.946Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "187b0b3f-e371-4255-a13f-78d356d58c8d", "cluster": "some-name", "storage": "minio", "allowed": false}
2026-04-09T02:37:45.995Z	INFO	Suspending backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "a4ee097b-883e-4e7b-b825-574dfbdc23fc", "job": "xb-backup3", "clusterStatus": "initializing", "readyPXC": 2}
2026-04-09T02:37:46.059Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "a4ee097b-883e-4e7b-b825-574dfbdc23fc", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:37:46.070Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "9523d1b4-75a2-45e0-b243-877891de10d0", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:37:51.070Z	INFO	Cluster is not ready for backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "c9733547-3aec-4bb6-8690-21aeb44de2c3", "cluster": "some-name", "reason": "unsafe.backupIfUnhealthy must be true to run backup on cluster with status initializing"}
2026-04-09T02:37:56.070Z	INFO	Backup didn't resume in suspendedDeadlineSeconds, failing the backup	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "715faac3-cd63-4a5f-9556-ed6c3e9befb8", "suspendedDeadlineSeconds": 10, "passedSeconds": 10.070971386}
2026-04-09T02:37:56.090Z	INFO	cleaning up suspended backup job	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "715faac3-cd63-4a5f-9556-ed6c3e9befb8", "cluster": "some-name"}
2026-04-09T02:37:56.168Z	INFO	Releasing backup lock	{"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "PerconaXtraDBClusterBackup": {"name":"backup3","namespace":"demand-backup-flow-control-17600"}, "namespace": "demand-backup-flow-control-17600", "name": "backup3", "reconcileID": "715faac3-cd63-4a5f-9556-ed6c3e9befb8", "cluster": "some-name", "lease": "pxc-some-name-backup-lock"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:313
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:438
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.23.1/pkg/internal/controller/controller.go:479
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:225
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:310
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n demand-backup-flow-control-17600 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.3JX8rVQmPj
++ mktemp
+ local LAST_ERR=/tmp/tmp.RYbK9YCx2Z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3JX8rVQmPj
perconaxtradbcluster.pxc.percona.com "some-name" deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.RYbK9YCx2Z
+ rm /tmp/tmp.3JX8rVQmPj /tmp/tmp.RYbK9YCx2Z
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.aRNnKqu4Mq
++ mktemp
+ local LAST_ERR=/tmp/tmp.uVfM9KTPcz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.aRNnKqu4Mq
perconaxtradbclusterbackup.pxc.percona.com "backup1" deleted from demand-backup-flow-control-17600 namespace
perconaxtradbclusterbackup.pxc.percona.com "backup2" deleted from demand-backup-flow-control-17600 namespace
perconaxtradbclusterbackup.pxc.percona.com "backup3" deleted from demand-backup-flow-control-17600 namespace
+ cat /tmp/tmp.uVfM9KTPcz
+ rm /tmp/tmp.aRNnKqu4Mq /tmp/tmp.uVfM9KTPcz
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ylDg2excX1
++ mktemp
+ local LAST_ERR=/tmp/tmp.DsnGIih0wI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ylDg2excX1
No resources found
+ cat /tmp/tmp.DsnGIih0wI
+ rm /tmp/tmp.ylDg2excX1 /tmp/tmp.DsnGIih0wI
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.QvuGcdEsD0
++ mktemp
+ local LAST_ERR=/tmp/tmp.uEjrBYmxcQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QvuGcdEsD0
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.uEjrBYmxcQ
+ rm /tmp/tmp.QvuGcdEsD0 /tmp/tmp.uEjrBYmxcQ
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.19.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ rm -rf /tmp/tmp.6GOpIwNc7Z
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
+ kubectl_bin delete --grace-period=0 --force=true namespace demand-backup-flow-control-17600
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.XR0NxT5iUU
+ local LAST_OUT=/tmp/tmp.jpXO4G21AB
++ mktemp
+ local LAST_ERR=/tmp/tmp.MbS5ZY2DMH
+ local exit_status=0
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ local LAST_ERR=/tmp/tmp.cgLcKdxo4A
+ set +e
+ local exit_status=0
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace demand-backup-flow-control-17600