Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/logs/demand-backup-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ main
+ create_infra demand-backup-26093
+ local ns=demand-backup-26093
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ grep -v NAMESPACE
+ kubectl patch pxc -n demand-backup-23039 demand-backup --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/demand-backup patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.Tf6aRMSXFb
++ mktemp
+ local LAST_ERR=/tmp/tmp.Js33kVtVHT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Tf6aRMSXFb
perconaxtradbcluster.pxc.percona.com "demand-backup" deleted from demand-backup-23039 namespace
+ cat /tmp/tmp.Js33kVtVHT
+ rm /tmp/tmp.Tf6aRMSXFb /tmp/tmp.Js33kVtVHT
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.XJdaay9T2e
++ mktemp
+ local LAST_ERR=/tmp/tmp.jVoTCwkNdP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.XJdaay9T2e
perconaxtradbclusterbackup.pxc.percona.com "on-demand-backup-pvc" deleted from demand-backup-23039 namespace
+ cat /tmp/tmp.jVoTCwkNdP
+ rm /tmp/tmp.XJdaay9T2e /tmp/tmp.jVoTCwkNdP
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.gQfULlI3S0
++ mktemp
+ local LAST_ERR=/tmp/tmp.bi2qUZtLCm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gQfULlI3S0
No resources found
+ cat /tmp/tmp.bi2qUZtLCm
+ rm /tmp/tmp.gQfULlI3S0 /tmp/tmp.bi2qUZtLCm
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ xargs kubectl delete ns
+ kubectl_bin get ns
++ mktemp
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_OUT=/tmp/tmp.q3gg7BxYVB
+ local LAST_OUT=/tmp/tmp.xnYaVO9dH5
++ mktemp
+ local LAST_ERR=/tmp/tmp.CU9GMgBD4R
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.pFD2kXrDMe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.q3gg7BxYVB
+ cat /tmp/tmp.CU9GMgBD4R
+ rm /tmp/tmp.q3gg7BxYVB /tmp/tmp.CU9GMgBD4R
+ return 0
namespace "demand-backup-23039" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xnYaVO9dH5
namespace "pxc-operator" deleted
+ cat /tmp/tmp.pFD2kXrDMe
+ rm /tmp/tmp.xnYaVO9dH5 /tmp/tmp.pFD2kXrDMe
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.C9qGx41MDT
++ mktemp
+ local LAST_ERR=/tmp/tmp.phHdm2N11Z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.C9qGx41MDT
namespace/pxc-operator created
+ cat /tmp/tmp.phHdm2N11Z
+ rm /tmp/tmp.C9qGx41MDT /tmp/tmp.phHdm2N11Z
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eEN9X3iOxE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3BeKGBxZFj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eEN9X3iOxE
++ cat /tmp/tmp.3BeKGBxZFj
++ rm /tmp/tmp.eEN9X3iOxE /tmp/tmp.3BeKGBxZFj
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.c7stnttWJP
++ mktemp
+ local LAST_ERR=/tmp/tmp.deJPvJHJF4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.c7stnttWJP
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5" modified.
+ cat /tmp/tmp.deJPvJHJF4
+ rm /tmp/tmp.c7stnttWJP /tmp/tmp.deJPvJHJF4
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.a7wMSizX2m
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q5In7pguCv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.a7wMSizX2m
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.Q5In7pguCv
+ rm /tmp/tmp.a7wMSizX2m /tmp/tmp.Q5In7pguCv
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5cCULsMJZ2
++ mktemp
+ local LAST_ERR=/tmp/tmp.XWd2hcTMgJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5cCULsMJZ2
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.XWd2hcTMgJ
+ rm /tmp/tmp.5cCULsMJZ2 /tmp/tmp.XWd2hcTMgJ
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eqlQtZwvqP
++ mktemp
+ local LAST_ERR=/tmp/tmp.zvouP3p8BH
+ local exit_status=0
++ seq 0 2
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eqlQtZwvqP
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.zvouP3p8BH
+ rm /tmp/tmp.eqlQtZwvqP /tmp/tmp.zvouP3p8BH
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.NPNml2NYMs
++ mktemp
+ local LAST_ERR=/tmp/tmp.Efv7q1y04Z
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NPNml2NYMs
pod/percona-xtradb-cluster-operator-b5f9c4897-r568j condition met
+ cat /tmp/tmp.Efv7q1y04Z
+ rm /tmp/tmp.NPNml2NYMs /tmp/tmp.Efv7q1y04Z
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zb3KEfOgkj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1WQAT6ZS3K
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zb3KEfOgkj
++ cat /tmp/tmp.1WQAT6ZS3K
++ rm /tmp/tmp.zb3KEfOgkj /tmp/tmp.1WQAT6ZS3K
++ return 0
+ wait_pod percona-xtradb-cluster-operator-b5f9c4897-r568j 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-b5f9c4897-r568j
+ local max_retry=480
+ local ns=pxc-operator
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo percona-xtradb-cluster-operator-b5f9c4897-r568j
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-b5f9c4897-r568j condition met
waiting for pod/percona-xtradb-cluster-operator-b5f9c4897-r568j to become Ready.Ok
+ sleep 3
+ create_namespace demand-backup-26093
+ local namespace=demand-backup-26093
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces demand-backup-26093'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces demand-backup-26093
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace demand-backup-26093
++ mktemp
+ local LAST_OUT=/tmp/tmp.KPIdOG85Qy
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.HHimQPn90t
++ mktemp
+ local LAST_ERR=/tmp/tmp.iuZqBdqbuM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ local LAST_ERR=/tmp/tmp.9Xg3liRoMg
+ local exit_status=0
+ xargs kubectl delete ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-26093
+ awk '{print$1}'
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-26093
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HHimQPn90t
+ cat /tmp/tmp.iuZqBdqbuM
+ rm /tmp/tmp.HHimQPn90t /tmp/tmp.iuZqBdqbuM
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace demand-backup-26093
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.KPIdOG85Qy
+ cat /tmp/tmp.9Xg3liRoMg
Error from server (NotFound): namespaces "demand-backup-26093" not found
+ rm /tmp/tmp.KPIdOG85Qy /tmp/tmp.9Xg3liRoMg
+ return 1
+ :
+ wait_for_delete namespace/demand-backup-26093
+ local res=namespace/demand-backup-26093
+ echo -n 'waiting for namespace/demand-backup-26093 to be deleted'
waiting for namespace/demand-backup-26093 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "demand-backup-26093" not found
+ desc 'create namespace demand-backup-26093'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace demand-backup-26093
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace demand-backup-26093
++ mktemp
+ local LAST_OUT=/tmp/tmp.tXjUNYXw4P
++ mktemp
+ local LAST_ERR=/tmp/tmp.7D1RD6RTtv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace demand-backup-26093
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tXjUNYXw4P
namespace/demand-backup-26093 created
+ cat /tmp/tmp.7D1RD6RTtv
+ rm /tmp/tmp.tXjUNYXw4P /tmp/tmp.7D1RD6RTtv
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.POu3CTcVrO
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KJpkabNfli
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.POu3CTcVrO
++ cat /tmp/tmp.KJpkabNfli
++ rm /tmp/tmp.POu3CTcVrO /tmp/tmp.KJpkabNfli
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5 --namespace=demand-backup-26093
++ mktemp
+ local LAST_OUT=/tmp/tmp.olpgmzRmxD
++ mktemp
+ local LAST_ERR=/tmp/tmp.RKjh00SQot
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5 --namespace=demand-backup-26093
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.olpgmzRmxD
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster5" modified.
+ cat /tmp/tmp.RKjh00SQot
+ rm /tmp/tmp.olpgmzRmxD /tmp/tmp.RKjh00SQot
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.GsLfO7Rzn6
++ mktemp
+ local LAST_ERR=/tmp/tmp.Re1lNDEGUn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GsLfO7Rzn6
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.Re1lNDEGUn
+ rm /tmp/tmp.GsLfO7Rzn6 /tmp/tmp.Re1lNDEGUn
+ return 0
+ cluster=demand-backup
+ spinup_pxc demand-backup /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/demand-backup.yml
+ local cluster=demand-backup
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/demand-backup.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.4yK1BuWOIh
++ mktemp
+ local LAST_ERR=/tmp/tmp.osQaRos0Fg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.4yK1BuWOIh
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.osQaRos0Fg
+ rm /tmp/tmp.4yK1BuWOIh /tmp/tmp.osQaRos0Fg
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-26093~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RZiHhRW2OA
++ mktemp
+ local LAST_ERR=/tmp/tmp.7FN0eqbWs0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RZiHhRW2OA
deployment.apps/pxc-client created
+ cat /tmp/tmp.7FN0eqbWs0
+ rm /tmp/tmp.RZiHhRW2OA /tmp/tmp.7FN0eqbWs0
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/demand-backup.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/demand-backup.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-26093~
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.7voHLlnFfy
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/demand-backup.yml
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_ERR=/tmp/tmp.aXi60zWTZO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7voHLlnFfy
perconaxtradbcluster.pxc.percona.com/demand-backup created
+ cat /tmp/tmp.aXi60zWTZO
+ rm /tmp/tmp.7voHLlnFfy /tmp/tmp.aXi60zWTZO
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy demand-backup
++ local target_cluster=demand-backup
+++ kubectl_bin get pxc demand-backup -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.S9mOKX8aWr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YsFp1LGyJE
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.S9mOKX8aWr
+++ cat /tmp/tmp.YsFp1LGyJE
+++ rm /tmp/tmp.S9mOKX8aWr /tmp/tmp.YsFp1LGyJE
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc demand-backup -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Y8B8gHBo0v
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2nx0MAgAa8
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc demand-backup -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.Y8B8gHBo0v
+++ cat /tmp/tmp.2nx0MAgAa8
+++ rm /tmp/tmp.Y8B8gHBo0v /tmp/tmp.2nx0MAgAa8
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo demand-backup-proxysql
++ return
+ local proxy=demand-backup-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-26093
++ mktemp
+ local LAST_OUT=/tmp/tmp.3lrGWgBM5v
++ mktemp
+ local LAST_ERR=/tmp/tmp.UXnnJ2IqzA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-26093
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-26093
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-26093
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.3lrGWgBM5v
+ cat /tmp/tmp.UXnnJ2IqzA
error: no matching resources found
+ rm /tmp/tmp.3lrGWgBM5v /tmp/tmp.UXnnJ2IqzA
+ return 1
+ true
+ wait_for_running demand-backup-proxysql 1
+ local name=demand-backup-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-proxysql-0 480
+ local pod=demand-backup-proxysql-0
+ local max_retry=480
+ local ns=
++ echo demand-backup-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/demand-backup-proxysql-0 condition met
waiting for pod/demand-backup-proxysql-0 to become Ready.Ok
+ wait_for_running demand-backup-pxc 3
+ local name=demand-backup-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-0 480
+ local pod=demand-backup-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo demand-backup-pxc-0
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-0 condition met
waiting for pod/demand-backup-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-1 480
+ local pod=demand-backup-pxc-1
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo demand-backup-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-1 condition met
waiting for pod/demand-backup-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod demand-backup-pxc-2 480
+ local pod=demand-backup-pxc-2
+ local max_retry=480
+ local ns=
++ echo demand-backup-pxc-2
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/demand-backup-pxc-2 condition met
waiting for pod/demand-backup-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc demand-backup -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L7Py9F7Wpo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vaYLUd2mhP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.L7Py9F7Wpo
++ cat /tmp/tmp.vaYLUd2mhP
++ rm /tmp/tmp.L7Py9F7Wpo /tmp/tmp.vaYLUd2mhP
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.mmrTIB9QIW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ghsJJe9Y9c
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.mmrTIB9QIW
++ cat /tmp/tmp.ghsJJe9Y9c
++ rm /tmp/tmp.mmrTIB9QIW /tmp/tmp.ghsJJe9Y9c
++ return 0
+ client_pod=pxc-client-59944c5bbf-pwdj9
+ wait_pod pxc-client-59944c5bbf-pwdj9
+ local pod=pxc-client-59944c5bbf-pwdj9
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-pwdj9
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-pwdj9 condition met
waiting for pod/pxc-client-59944c5bbf-pwdj9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h demand-backup-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HHPey4yDml
+++ mktemp
++ local LAST_ERR=/tmp/tmp.15D26R1o14
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HHPey4yDml
++ cat /tmp/tmp.15D26R1o14
++ rm /tmp/tmp.HHPey4yDml /tmp/tmp.15D26R1o14
++ return 0
+ client_pod=pxc-client-59944c5bbf-pwdj9
+ wait_pod pxc-client-59944c5bbf-pwdj9
+ local pod=pxc-client-59944c5bbf-pwdj9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-pwdj9
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-pwdj9 condition met
waiting for pod/pxc-client-59944c5bbf-pwdj9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-0.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rvtc6RaMBM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y9YUjr8l8c
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rvtc6RaMBM
++ cat /tmp/tmp.y9YUjr8l8c
++ rm /tmp/tmp.rvtc6RaMBM /tmp/tmp.y9YUjr8l8c
++ return 0
+ client_pod=pxc-client-59944c5bbf-pwdj9
+ wait_pod pxc-client-59944c5bbf-pwdj9
+ local pod=pxc-client-59944c5bbf-pwdj9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-pwdj9
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-pwdj9 condition met
waiting for pod/pxc-client-59944c5bbf-pwdj9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.HYf9PiPehK/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.HYf9PiPehK/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-1.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ga6Kfqdk1E
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VMYTSRiZvM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ga6Kfqdk1E
++ cat /tmp/tmp.VMYTSRiZvM
++ rm /tmp/tmp.Ga6Kfqdk1E /tmp/tmp.VMYTSRiZvM
++ return 0
+ client_pod=pxc-client-59944c5bbf-pwdj9
+ wait_pod pxc-client-59944c5bbf-pwdj9
+ local pod=pxc-client-59944c5bbf-pwdj9
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-pwdj9
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-pwdj9 condition met
waiting for pod/pxc-client-59944c5bbf-pwdj9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.HYf9PiPehK/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.HYf9PiPehK/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h demand-backup-pxc-2.demand-backup-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nxLUZMJIGs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6t8QXXCbIA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nxLUZMJIGs
++ cat /tmp/tmp.6t8QXXCbIA
++ rm /tmp/tmp.nxLUZMJIGs /tmp/tmp.6t8QXXCbIA
++ return 0
+ client_pod=pxc-client-59944c5bbf-pwdj9
+ wait_pod pxc-client-59944c5bbf-pwdj9
+ local pod=pxc-client-59944c5bbf-pwdj9
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-pwdj9
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-pwdj9 condition met
waiting for pod/pxc-client-59944c5bbf-pwdj9 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.HYf9PiPehK/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/compare/select-1.sql /tmp/tmp.HYf9PiPehK/select-1.sql
++ is_keyring_plugin_in_use demand-backup
++ local cluster=demand-backup
++ egrep -o 'early-plugin-load=keyring_\w+.so'
++ kubectl_bin exec -it demand-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7X1mVmmLVQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qcBW4iOJVf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it demand-backup-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7X1mVmmLVQ
++ cat /tmp/tmp.qcBW4iOJVf
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.7X1mVmmLVQ /tmp/tmp.qcBW4iOJVf
++ return 0
+ '[' '' ']'
+ start_minio demand-backup-ssl
+ deploy_helm demand-backup-26093
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=demand-backup-ssl
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n demand-backup-ssl ]]
+ endpoint=https://minio-service:9000
+ minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key)
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=demand-backup-ssl --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=demand-backup-ssl --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Nov 12 10:34:18 2025
NAMESPACE: demand-backup-26093
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.demand-backup-26093.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace demand-backup-26093 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace demand-backup-26093

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace demand-backup-26093 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace demand-backup-26093 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0QACftKr3d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZEZZD0WvNZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0QACftKr3d
++ cat /tmp/tmp.ZEZZD0WvNZ
++ rm /tmp/tmp.0QACftKr3d /tmp/tmp.ZEZZD0WvNZ
++ return 0
+ MINIO_POD=minio-service-f46fd9c97-j2j4t
+ wait_pod minio-service-f46fd9c97-j2j4t
+ local pod=minio-service-f46fd9c97-j2j4t
+ local max_retry=480
+ local ns=
++ echo minio-service-f46fd9c97-j2j4t
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minio-service-f46fd9c97-j2j4t condition met
waiting for pod/minio-service-f46fd9c97-j2j4t to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.EeDj3YeaqO
++ mktemp
+ local LAST_ERR=/tmp/tmp.U6CahJTnGm
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EeDj3YeaqO
pod "aws-cli" deleted from demand-backup-26093 namespace
+ cat /tmp/tmp.U6CahJTnGm
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
+ rm /tmp/tmp.EeDj3YeaqO /tmp/tmp.U6CahJTnGm
+ return 0
+ run_backup demand-backup on-demand-backup-pvc
+ local cluster=demand-backup
+ local backup=on-demand-backup-pvc
+ log 'run pxc-backup/on-demand-backup-pvc'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2025-11-12T10:35:32+0000]' run pxc-backup/on-demand-backup-pvc
[2025-11-12T10:35:32+0000] run pxc-backup/on-demand-backup-pvc
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tRvzRpntGj
++ mktemp
+ local LAST_ERR=/tmp/tmp.fBFLPeMlyP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup/conf/on-demand-backup-pvc.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tRvzRpntGj
perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-pvc created
+ cat /tmp/tmp.fBFLPeMlyP
+ rm /tmp/tmp.tRvzRpntGj /tmp/tmp.fBFLPeMlyP
+ return 0
+ wait_backup on-demand-backup-pvc
+ local backup=on-demand-backup-pvc
+ local status=Succeeded
+ set +o xtrace
waiting for pxc-backup/on-demand-backup-pvc to reach Succeeded state........................................................................................................................................................................................................................................................................................................................................................................2025-11-12T10:28:07.788Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.31.13-gke.1231000"}
2025-11-12T10:28:07.788Z	INFO	setup	Manager starting up	{"gitCommit": "269f3694d9306fde7f2ffd7864a3f22b0ef247c5", "gitBranch": "PR-2234-269f3694", "buildTime": "2025-11-12T09:55:34Z", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
2025-11-12T10:28:07.791Z	INFO	setup	Registering Components.
2025-11-12T10:28:08.455Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2025-11-12T10:28:08.455Z	INFO	setup	Starting the Cmd.
2025-11-12T10:28:08.455Z	INFO	controller-runtime.metrics	Starting metrics server
2025-11-12T10:28:08.455Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2025-11-12T10:28:08.455Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-11-12T10:28:08.455Z	INFO	controller-runtime.webhook	Starting webhook server
2025-11-12T10:28:08.464Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2025-11-12T10:28:08.464Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2025-11-12T10:28:08.464Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2025-11-12T10:28:08.565Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2025-11-12T10:28:08.597Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2025-11-12T10:28:08.597Z	DEBUG	events	percona-xtradb-cluster-operator-b5f9c4897-r568j_6a19a700-88e2-4005-b12c-f2d531e150c2 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"77375450-86e4-4000-be5e-71a8e3e606ba","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1762943288588175009"}, "reason": "LeaderElection"}
2025-11-12T10:28:08.597Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.Secret"}
2025-11-12T10:28:08.597Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2025-11-12T10:28:08.597Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2025-11-12T10:28:08.597Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2025-11-12T10:28:08.697Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2025-11-12T10:28:08.698Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2025-11-12T10:28:08.697Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2025-11-12T10:28:08.698Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2025-11-12T10:28:08.698Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2025-11-12T10:28:08.698Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2025-11-12T10:29:01.371Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1f311cde-57c7-4f8b-81db-5b0946e3bca6", "version": "1.19.0"}
2025-11-12T10:29:02.707Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1f311cde-57c7-4f8b-81db-5b0946e3bca6", "object": "auto-demand-backup-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:29:02.746Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1f311cde-57c7-4f8b-81db-5b0946e3bca6", "object": "demand-backup-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:29:02.785Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1f311cde-57c7-4f8b-81db-5b0946e3bca6", "object": "demand-backup-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:29:03.345Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1f311cde-57c7-4f8b-81db-5b0946e3bca6", "error": "failed to deploy proxysql: updatePod for proxysql: reconcile config: reconcile custom config: create or update config map: configmaps \"demand-backup-pxc\" already exists", "errorVerbose": "configmaps \"demand-backup-pxc\" already exists\ncreate or update config map\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileCustomConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:115\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:31\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile custom config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:33\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:578\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:596\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296
2025-11-12T10:29:03.450Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-11-12T10:29:03.508Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-11-12T10:29:03.560Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:29:03.600Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:29:03.663Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:29:03.756Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "dc81b7bb-8ba6-4fbb-ab33-d7740b1bdacd", "object": "demand-backup-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:29:04.935Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1ef3757f-ed0e-459a-9600-02e62e2575fe", "object": "demand-backup-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-11-12T10:29:05.016Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1ef3757f-ed0e-459a-9600-02e62e2575fe", "object": "demand-backup-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-11-12T10:30:21.324Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9", "user": "operator"}
2025-11-12T10:30:21.389Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9", "user": "monitor"}
2025-11-12T10:30:21.456Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9"}
2025-11-12T10:30:21.501Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9"}
2025-11-12T10:30:21.549Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9", "user": "xtrabackup"}
2025-11-12T10:30:21.591Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9"}
2025-11-12T10:30:21.629Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9", "user": "replication"}
2025-11-12T10:30:21.637Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "84440365-6c7d-41df-bb83-cc267aae02a9", "err": "get primary pxc pod: not found"}
2025-11-12T10:30:26.238Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "8d7d052b-1fb7-4aa4-a3d7-d723c3583f8c", "err": "get primary pxc pod: not found"}
2025-11-12T10:30:31.360Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "212d502f-beba-42d7-b75e-50939923cc5e", "err": "get primary pxc pod: not found"}
2025-11-12T10:30:36.556Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3c8758b5-9ce0-43ac-ab90-41ab5383fda3", "err": "get primary pxc pod: not found"}
2025-11-12T10:32:47.994Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e922fce9-8022-47ba-b6c0-d73387663bcb", "user": "root"}
2025-11-12T10:32:48.108Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e922fce9-8022-47ba-b6c0-d73387663bcb", "new version": "8.0.43-34.1"}
2025-11-12T10:32:49.875Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e922fce9-8022-47ba-b6c0-d73387663bcb"}
2025-11-12T10:32:54.684Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "9fa3f8c9-fa7b-49a1-a53c-4e4631638f5e"}
2025-11-12T10:33:00.053Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "58e1cb20-5077-4cdf-b477-833b358e00f9"}
2025-11-12T10:33:05.288Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "390e2aa9-7db1-4e75-b682-7e825c5ae309"}
2025-11-12T10:33:10.658Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e7c494a7-3617-4eba-affb-20efc25d1fe9"}
2025-11-12T10:33:16.045Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "87ee4513-37cb-4733-9fcd-c3428d24edf7"}
2025-11-12T10:33:21.272Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1c08a224-9e0e-44df-94dd-6c1b5f67a253"}
2025-11-12T10:33:26.378Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e0908568-8775-4e46-a8ea-c8d90f4c63fd"}
2025-11-12T10:33:31.853Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "af248566-189e-45cd-97da-a535eb9e8f1c"}
2025-11-12T10:33:37.174Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f293d97e-22ec-4b20-add3-1da22f57f45d"}
2025-11-12T10:33:42.542Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7ce16274-0a03-407a-a8f1-1fa8ab6b18b6"}
2025-11-12T10:33:47.846Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6b68a767-88cd-4c1d-a1df-b044a3c48fd1"}
2025-11-12T10:33:53.160Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "099f6341-06de-4be1-bb44-a7106512afec"}
2025-11-12T10:33:58.753Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e6ff8cc7-2246-4b5f-ad77-cd93299ab993"}
2025-11-12T10:34:04.075Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2d32f8e9-5fb0-4b84-b69a-b8b3b325c8d4"}
2025-11-12T10:34:09.293Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "be490681-0295-414d-89c3-c73871dc12e2"}
2025-11-12T10:34:14.679Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "c90a0001-392e-4acc-93db-1c23d9474e6f"}
2025-11-12T10:34:20.056Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "9d0f701a-db8e-4e4f-a635-f87d90bf975f"}
2025-11-12T10:34:25.393Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0eda8587-2111-4996-83a4-cecd3067e641"}
2025-11-12T10:34:30.756Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6aa77fa8-8c1e-47c3-b4b5-d5882df15c28"}
2025-11-12T10:34:36.328Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7263882a-3bf5-49f3-b70f-289ad0322ef6"}
2025-11-12T10:34:41.189Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5f64dbcc-690c-40d1-90db-6d50b0d63104"}
2025-11-12T10:34:46.842Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6b728cea-2316-4bdf-8d49-eb3ee37060df"}
2025-11-12T10:34:52.282Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6e8b65c1-e6f4-4130-be5a-36022ab3954e"}
2025-11-12T10:34:57.386Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f97785a9-ab22-4900-b294-8c68a0a759b2"}
2025-11-12T10:35:02.542Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6e411866-7dbb-4443-b1de-5a2991433f74"}
2025-11-12T10:35:08.547Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "351215f5-087a-4e94-93a7-a893c504ebd5"}
2025-11-12T10:35:13.848Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "c2af2315-8a9e-4c24-8683-32b3edf94e1a"}
2025-11-12T10:35:19.248Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "883fa080-eb2c-4129-a84e-36daf21d05d2"}
2025-11-12T10:35:24.542Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "a4f51718-7193-4971-a404-46debed82df7"}
2025-11-12T10:35:29.781Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "45ab7d1b-5da7-4673-b136-2dc5f3cfeb9f"}
2025-11-12T10:35:34.959Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "b6d901a0-830a-4d30-8d5f-802e715d0306", "cluster": "demand-backup", "storage": "pvc", "allowed": false}
2025-11-12T10:35:35.083Z	INFO	Creating a new volume for backup	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "b6d901a0-830a-4d30-8d5f-802e715d0306", "Namespace": "demand-backup-26093", "Name": "xb-on-demand-backup-pvc"}
2025-11-12T10:35:35.144Z	INFO	Created a new backup job	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "b6d901a0-830a-4d30-8d5f-802e715d0306", "namespace": "demand-backup-26093", "name": "xb-on-demand-backup-pvc"}
2025-11-12T10:35:35.144Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "1603d622-448d-4ce9-843c-f5d7d5ccb5ac", "cluster": "demand-backup", "storage": "pvc", "allowed": false}
2025-11-12T10:35:35.166Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f228f711-8866-46d6-a387-065d3b2b5c78"}
2025-11-12T10:35:35.261Z	INFO	Releasing backup lock	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "1603d622-448d-4ce9-843c-f5d7d5ccb5ac", "cluster": "demand-backup", "storage": "pvc", "lease": "pxc-demand-backup-backup-lock"}
2025-11-12T10:35:35.303Z	ERROR	Reconciler error	{"controller": "pxcbackup-controller", "namespace": "demand-backup-26093", "name": "on-demand-backup-pvc", "reconcileID": "1603d622-448d-4ce9-843c-f5d7d5ccb5ac", "error": "create backup job: create backup job: jobs.batch \"xb-on-demand-backup-pvc\" already exists", "errorVerbose": "jobs.batch \"xb-on-demand-backup-pvc\" already exists\ncreate backup job\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).createBackupJob\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:383\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:291\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate backup job\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:293\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296
2025-11-12T10:35:40.436Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "671e030f-d905-4c17-9db0-d6c3ecf6ad41"}
2025-11-12T10:35:45.556Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0e1f2f8a-97f2-4d56-9ec9-4bc6309e2232"}
2025-11-12T10:35:51.039Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "58833852-1a3f-41e6-87e1-6eac7b861f71"}
2025-11-12T10:35:56.347Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3ef732b3-971e-404a-88c5-ebff35bd4a0b"}
2025-11-12T10:36:01.640Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "ecfa974d-cfc9-4789-aa90-bf4393c94369"}
2025-11-12T10:36:07.259Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2edec80e-e92f-45d4-acd9-17f5caa2d21f"}
2025-11-12T10:36:12.264Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6f8b375b-a56a-4928-b151-760a304233b0"}
2025-11-12T10:36:17.560Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5b19507c-fa9e-4fda-a3d0-f13f1fd8fbaa"}
2025-11-12T10:36:23.244Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b205ed43-8a78-4c63-9f75-a950f61c47c8"}
2025-11-12T10:36:28.539Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0b0bebad-1986-46e0-b8c0-88ada8d59ad5"}
2025-11-12T10:36:33.888Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "646e703a-cba6-4a4e-bfd8-e81a5f7d71c2"}
2025-11-12T10:36:39.147Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2871f953-45a0-40ac-9aaf-37ba029763cf"}
2025-11-12T10:36:44.556Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2d66aed7-8f2a-4893-862e-d6ba088e5a4e"}
2025-11-12T10:36:50.294Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3aa80bf7-78b1-4733-8ab7-cc02c03eb18a"}
2025-11-12T10:36:55.392Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5774e5de-f4b0-42c6-ab1e-9ab6ec2f45c3"}
2025-11-12T10:37:00.793Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "cb409658-477c-4a0b-95be-fdc327f40602"}
2025-11-12T10:37:06.343Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "847e9f0d-18ad-4e22-9a78-330df85361ac"}
2025-11-12T10:37:11.462Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "87dbbae0-d88b-4c86-bb15-5ca632a8ee9f"}
2025-11-12T10:37:16.838Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "24d5feec-2f7d-4789-b433-1d261130e5bb"}
2025-11-12T10:37:22.266Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "761e67fc-8c06-492f-8811-bf8a689ac941"}
2025-11-12T10:37:27.469Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1dbf407a-0c5e-4027-ad35-17fc409573fd"}
2025-11-12T10:37:32.845Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "db00f2b2-7a3e-49c9-877a-c45354154400"}
2025-11-12T10:37:38.163Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "94428a49-73e3-4ff5-b60d-126c2434de38"}
2025-11-12T10:37:43.759Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "a698a717-a00d-4170-9255-be06a627908e"}
2025-11-12T10:37:49.160Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "a66acbbc-5053-44a7-916b-bce643519b2d"}
2025-11-12T10:37:54.255Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "89827159-0a18-4bba-81b9-914e5a09b6c5"}
2025-11-12T10:37:59.461Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7c05d649-f69a-4490-8d23-02336ff61223"}
2025-11-12T10:38:04.980Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "9b7bd160-f397-4f6a-a880-23ad955501f5"}
2025-11-12T10:38:10.257Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3afc8416-7cf9-4b4b-83cb-9447a82a8488"}
2025-11-12T10:38:15.659Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "92f53f27-f7d8-40de-a52d-114e040ab8c6"}
2025-11-12T10:38:21.138Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b39f21cb-745b-420c-8079-a0b63efb3ad3"}
2025-11-12T10:38:26.451Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0d923d48-b35f-47e1-adb8-b1570023c8ed"}
2025-11-12T10:38:31.747Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "24daf1fc-bbda-4b94-a051-58bafbbd9033"}
2025-11-12T10:38:37.263Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2c1b10c1-3155-4105-a2d4-b85b5c0da42d"}
2025-11-12T10:38:42.148Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "90edd7a8-7080-46d9-8b36-36b110eb7f06"}
2025-11-12T10:38:47.693Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "58276af3-fa93-4046-a458-e5c9578fc9bc"}
2025-11-12T10:38:53.164Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6f61f6d3-3413-4a6f-b43b-94d41f19d1cc"}
2025-11-12T10:38:58.657Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "115cb51b-84bc-4c11-8439-3b8834315cb2"}
2025-11-12T10:39:03.959Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4998f7f9-0044-45d8-bc4c-1c81b173f897"}
2025-11-12T10:39:09.249Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f9f4762c-b4be-4cbf-9d88-853355dcba14"}
2025-11-12T10:39:14.554Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "67e6e8f0-b6a8-414a-bf4c-89b46ac971ab"}
2025-11-12T10:39:19.855Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0765ca49-56fd-4741-bf59-66e23a6ea2b5"}
2025-11-12T10:39:24.949Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7feb1013-86d8-4692-978e-ce31cdef1318"}
2025-11-12T10:39:30.438Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4ab681b3-44f2-437f-9fac-286311b63e71"}
2025-11-12T10:39:35.881Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "69167aff-4349-466e-ad3f-e7fa9feb177b"}
2025-11-12T10:39:41.244Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6656f97e-4f8b-4d97-9e92-4a6529d529c5"}
2025-11-12T10:39:46.651Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "9402c385-b0d6-4aed-80e6-bce769c9e42b"}
2025-11-12T10:39:52.143Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "154ee459-1caf-4eeb-b35e-6fc50eb5e521"}
2025-11-12T10:39:57.245Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "d1fc21b1-4a70-4622-9e42-4679b9b24f85"}
2025-11-12T10:40:02.567Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "a20f5478-ba74-4072-8768-b6afec6de870"}
2025-11-12T10:40:08.285Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "aa0c393b-03ea-43d0-b9c1-c7931ef74dec"}
2025-11-12T10:40:13.691Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "538a2658-bb02-4ba2-920c-445910db0100"}
2025-11-12T10:40:19.064Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "07da778a-6b78-4332-976d-7e7c981a333a"}
2025-11-12T10:40:24.444Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f4ca2470-ee25-4b96-87a0-268e841d117e"}
2025-11-12T10:40:29.695Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5b6d14a0-0ed9-47c3-98ed-b2e9a8e4c473"}
2025-11-12T10:40:35.049Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "53661ba7-b558-44f5-a056-c761b042a579"}
2025-11-12T10:40:40.345Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "c669c806-f715-4f16-be52-4c9b56daacc9"}
2025-11-12T10:40:45.691Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "cdb511cd-8854-47ff-ae5a-5be549aba3cc"}
2025-11-12T10:40:51.057Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "35e3c072-a9fc-47d3-9bc2-5fa2c3b02fd3"}
2025-11-12T10:40:56.354Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "03aeddc4-297c-4c18-b3ee-012a68ecac1e"}
2025-11-12T10:41:01.665Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "97c3f85e-99ef-4166-b894-674b5807d50b"}
2025-11-12T10:41:07.267Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6cba1475-799d-40ed-b168-6cdcc499a9ab"}
2025-11-12T10:41:12.264Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "552541e5-e749-41ad-85c9-351a12570af6"}
2025-11-12T10:41:17.652Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "57c52b54-a7ea-48db-808d-e5519694702e"}
2025-11-12T10:41:23.067Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7f863d67-db6c-4fab-b38e-0da1be6d067e"}
2025-11-12T10:41:28.448Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "50450154-e335-4cbe-bf66-0ea550adbd4d"}
2025-11-12T10:41:33.765Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "bcd4f2e5-99fd-43c0-b052-8397870f1607"}
2025-11-12T10:41:39.063Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e730bd3a-2071-4e54-83e6-e03c9e3bdd29"}
2025-11-12T10:41:44.347Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "de55b670-8003-4ae7-a83a-7275c3f75100"}
2025-11-12T10:41:49.770Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "8b0e12ab-03e7-4698-94cd-72947b553138"}
2025-11-12T10:41:54.987Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "ef3d6172-daec-4846-a55f-f9d30cb8f8c4"}
2025-11-12T10:42:00.344Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4b52b2fa-316e-4b26-964e-8c3e9ad23cb6"}
2025-11-12T10:42:05.846Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "d190710b-ecf6-45dc-9070-57921896552d"}
2025-11-12T10:42:11.258Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "9bac7257-be0b-439e-8a99-142394a50a74"}
2025-11-12T10:42:16.538Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "28adbe13-8976-45c6-9ac2-6c55e7732195"}
2025-11-12T10:42:21.875Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "60111400-4a21-46b6-be17-587ae821b7c8"}
2025-11-12T10:42:26.949Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "98bc9309-2768-4c73-9411-b0846cd46820"}
2025-11-12T10:42:32.473Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "d124ff14-fede-4056-8726-3c2d95ff4c09"}
2025-11-12T10:42:37.893Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "39f6c686-cf09-4423-8cb2-9684dd9e96ab"}
2025-11-12T10:42:43.078Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b313588a-fd45-4737-9dc0-7e177b5b9f84"}
2025-11-12T10:42:48.362Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "0a615bae-dbf7-4cb6-a29e-2f6d1a8dff7e"}
2025-11-12T10:42:53.663Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4f39dfad-2796-4c40-b9ed-87445d1f7818"}
2025-11-12T10:42:58.836Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6818329c-6a4c-48f2-a4d5-2a85814e29e0"}
2025-11-12T10:43:04.354Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e824f75f-0da9-4cd8-98d8-1893cbe4807a"}
2025-11-12T10:43:09.648Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7170768d-457d-40cb-bc13-d75f46aa6eed"}
2025-11-12T10:43:14.950Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6180fbe7-093c-4b6d-b99d-c780efc82c5c"}
2025-11-12T10:43:20.241Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "bb658c57-dc3d-45cc-abef-9ef2ac9ee72d"}
2025-11-12T10:43:25.647Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "254c4e81-8177-436e-afa1-91dbfbd85d30"}
2025-11-12T10:43:30.844Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4a325bb4-d03b-4e39-9b95-ff949926c99c"}
2025-11-12T10:43:36.078Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "6f385b37-dabe-4114-8aac-04ea2fe40c10"}
2025-11-12T10:43:41.362Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "f3ef8551-0549-41f2-bd39-05d68dba6471"}
2025-11-12T10:43:46.596Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "08903f87-ce22-497b-9817-39ada710e8be"}
2025-11-12T10:43:52.155Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3c1314ef-d434-4065-881f-f2bc95637f2f"}
2025-11-12T10:43:57.258Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3c8e28e7-b4e0-432d-8785-0b8354b936ed"}
2025-11-12T10:44:02.344Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "caca2a5d-3680-424f-9280-00d7320e06f6"}
2025-11-12T10:44:08.161Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "44457593-429d-436f-9627-9d4798aaf948"}
2025-11-12T10:44:13.661Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "95dc1a81-5b3d-4abc-af90-81ac2e29590c"}
2025-11-12T10:44:19.067Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "afa04e7d-9bf4-4bf4-8e38-3c8bd044d29f"}
2025-11-12T10:44:24.444Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "538a9b38-613d-42ee-aa8b-1e766e888795"}
2025-11-12T10:44:29.654Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "86d4a52a-b7f1-4459-b2b9-771a63335f58"}
2025-11-12T10:44:35.047Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "e90cf139-617c-4213-98da-dd20421a7840"}
2025-11-12T10:44:40.358Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "36b47c3e-08f5-4fdd-bbfb-22f32d50ce0d"}
2025-11-12T10:44:45.451Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "a800aceb-0e77-47df-b640-3f236b57917a"}
2025-11-12T10:44:50.966Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7a01cf24-38df-4b2c-8dde-14129f9b7d77"}
2025-11-12T10:44:56.358Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "194e4ded-7f89-4789-9825-fcd36bc2be07"}
2025-11-12T10:45:01.696Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "359a0e9d-4d10-4523-be23-630c053c6964"}
2025-11-12T10:45:07.486Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "41306eac-7d17-495c-92d3-71482ade3601"}
2025-11-12T10:45:12.362Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "64ed709a-e2ad-4e92-8c2d-7f4a25d16aa1"}
2025-11-12T10:45:17.650Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b19570c4-d074-4df0-abc3-b191d7cb0717"}
2025-11-12T10:45:23.258Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "7b0fc64f-b9e6-4780-8ff0-f15c38a631d4"}
2025-11-12T10:45:28.648Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "36d57d43-066d-4248-8fbe-d6db93e97fb7"}
2025-11-12T10:45:33.888Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "25eca588-41e2-4492-9b6b-a298e7a87e6c"}
2025-11-12T10:45:39.246Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "4e3120ac-e0c7-415d-ad58-e19d9706fb37"}
2025-11-12T10:45:44.557Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5e939325-53c3-4d3e-9aa7-ee890601f4b8"}
2025-11-12T10:45:49.839Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "01bbf61a-1232-4917-9228-1cc70e60046f"}
2025-11-12T10:45:54.941Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "5b452316-ff07-4c06-b4c4-1ec9b4490227"}
2025-11-12T10:46:00.461Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "2b84b89f-1238-45c0-932c-db1ddf24cfd5"}
2025-11-12T10:46:06.062Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b04d12cb-7c4a-4cf5-9016-0d664085f364"}
2025-11-12T10:46:11.390Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b5124ae2-4f66-41ee-a0db-2e2b702e596f"}
2025-11-12T10:46:16.743Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "1c90d127-81fa-4589-88af-7c59cd0c34dd"}
2025-11-12T10:46:22.188Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "b640ff0b-df6a-4de4-9cf4-0415e6dc79ce"}
2025-11-12T10:46:27.356Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "19104af1-4e8e-4917-97ef-ad8759254c02"}
2025-11-12T10:46:32.662Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "demand-backup-26093", "name": "demand-backup", "reconcileID": "3ea01e63-3265-4056-9518-9cae6ec6a761"}
max retry count 360 reached. something went wrong with operator or kubernetes cluster