Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/logs/backup-storage-tls-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ GTID_PATTERN='[A-F0-9a-f]{8}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{12}:[0-9]+'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ main
+ create_infra backup-storage-tls-1228
+ local ns=backup-storage-tls-1228
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n backup-storage-tls-24311 test-cluster --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/test-cluster patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.bSi2sg2f3N
++ mktemp
+ local LAST_ERR=/tmp/tmp.GxhwEsI37L
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.bSi2sg2f3N
perconaxtradbcluster.pxc.percona.com "test-cluster" deleted from backup-storage-tls-24311 namespace
+ cat /tmp/tmp.GxhwEsI37L
+ rm /tmp/tmp.bSi2sg2f3N /tmp/tmp.GxhwEsI37L
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.HCNukjrul2
++ mktemp
+ local LAST_ERR=/tmp/tmp.kSaqPs9qxb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HCNukjrul2
perconaxtradbclusterbackup.pxc.percona.com "on-demand-backup" deleted from backup-storage-tls-24311 namespace
+ cat /tmp/tmp.kSaqPs9qxb
+ rm /tmp/tmp.HCNukjrul2 /tmp/tmp.kSaqPs9qxb
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.1QeIbWiTUQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.7qwrjYlzfF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1QeIbWiTUQ
No resources found
+ cat /tmp/tmp.7qwrjYlzfF
+ rm /tmp/tmp.1QeIbWiTUQ /tmp/tmp.7qwrjYlzfF
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ xargs kubectl delete ns
+ awk '{print$1}'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.0Oy7xt1MWo
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.iBdZJKsKXu
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.nYwUQIuQ9q
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.qTx9jFP3JH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.0Oy7xt1MWo
+ cat /tmp/tmp.nYwUQIuQ9q
+ rm /tmp/tmp.0Oy7xt1MWo /tmp/tmp.nYwUQIuQ9q
+ return 0
namespace "backup-storage-tls-24311" deleted
namespace "cert-manager" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.iBdZJKsKXu
namespace "pxc-operator" deleted
+ cat /tmp/tmp.qTx9jFP3JH
+ rm /tmp/tmp.iBdZJKsKXu /tmp/tmp.qTx9jFP3JH
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.UIPxQtpghj
++ mktemp
+ local LAST_ERR=/tmp/tmp.p0UIzv2RWi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UIPxQtpghj
namespace/pxc-operator created
+ cat /tmp/tmp.p0UIzv2RWi
+ rm /tmp/tmp.UIPxQtpghj /tmp/tmp.p0UIzv2RWi
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ttalKwE8Qp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lhRrqGKUdx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ttalKwE8Qp
++ cat /tmp/tmp.lhRrqGKUdx
++ rm /tmp/tmp.ttalKwE8Qp /tmp/tmp.lhRrqGKUdx
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.BF0DIIIUMl
++ mktemp
+ local LAST_ERR=/tmp/tmp.os2V1TX03r
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BF0DIIIUMl
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1" modified.
+ cat /tmp/tmp.os2V1TX03r
+ rm /tmp/tmp.BF0DIIIUMl /tmp/tmp.os2V1TX03r
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.SAdDqCBgTi
++ mktemp
+ local LAST_ERR=/tmp/tmp.eAm9i6G8Dr
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SAdDqCBgTi
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.eAm9i6G8Dr
+ rm /tmp/tmp.SAdDqCBgTi /tmp/tmp.eAm9i6G8Dr
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZXnjfSk9b9
++ mktemp
+ local LAST_ERR=/tmp/tmp.ywtzq2WRzJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ZXnjfSk9b9
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.ywtzq2WRzJ
+ rm /tmp/tmp.ZXnjfSk9b9 /tmp/tmp.ywtzq2WRzJ
+ return 0
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ kubectl_bin apply -f -
++ mktemp
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-operator.yaml
+ local LAST_OUT=/tmp/tmp.fkoNKvlRTd
++ mktemp
+ local LAST_ERR=/tmp/tmp.QR10XJQy6b
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fkoNKvlRTd
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.QR10XJQy6b
+ rm /tmp/tmp.fkoNKvlRTd /tmp/tmp.QR10XJQy6b
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.2tMsPNjLVD
++ mktemp
+ local LAST_ERR=/tmp/tmp.6uzwWFSeHR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2tMsPNjLVD
pod/percona-xtradb-cluster-operator-b5f9c4897-fdrp8 condition met
+ cat /tmp/tmp.6uzwWFSeHR
+ rm /tmp/tmp.2tMsPNjLVD /tmp/tmp.6uzwWFSeHR
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j3c7y8SAI1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6wOZjV5Ens
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.j3c7y8SAI1
++ cat /tmp/tmp.6wOZjV5Ens
++ rm /tmp/tmp.j3c7y8SAI1 /tmp/tmp.6wOZjV5Ens
++ return 0
+ wait_pod percona-xtradb-cluster-operator-b5f9c4897-fdrp8 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-b5f9c4897-fdrp8
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-b5f9c4897-fdrp8
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-b5f9c4897-fdrp8 condition met
waiting for pod/percona-xtradb-cluster-operator-b5f9c4897-fdrp8 to become Ready.Ok
+ sleep 3
+ create_namespace backup-storage-tls-1228
+ local namespace=backup-storage-tls-1228
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ sed s/NAMESPACE//
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces backup-storage-tls-1228'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces backup-storage-tls-1228
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace backup-storage-tls-1228
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.PTp4snWGOS
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.MfBVJutMum
+ local exit_status=0
+ local LAST_OUT=/tmp/tmp.8ZhiZtcv8K
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace backup-storage-tls-1228
+ local LAST_ERR=/tmp/tmp.XxHNlWzad7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8ZhiZtcv8K
+ cat /tmp/tmp.XxHNlWzad7
+ rm /tmp/tmp.8ZhiZtcv8K /tmp/tmp.XxHNlWzad7
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace backup-storage-tls-1228
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace backup-storage-tls-1228
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.PTp4snWGOS
+ cat /tmp/tmp.MfBVJutMum
Error from server (NotFound): namespaces "backup-storage-tls-1228" not found
+ rm /tmp/tmp.PTp4snWGOS /tmp/tmp.MfBVJutMum
+ return 1
+ :
+ wait_for_delete namespace/backup-storage-tls-1228
+ local res=namespace/backup-storage-tls-1228
+ echo -n 'waiting for namespace/backup-storage-tls-1228 to be deleted'
waiting for namespace/backup-storage-tls-1228 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "backup-storage-tls-1228" not found
+ desc 'create namespace backup-storage-tls-1228'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace backup-storage-tls-1228
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace backup-storage-tls-1228
++ mktemp
+ local LAST_OUT=/tmp/tmp.8Ce26DgL02
++ mktemp
+ local LAST_ERR=/tmp/tmp.Tw3OrzyLp1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace backup-storage-tls-1228
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8Ce26DgL02
namespace/backup-storage-tls-1228 created
+ cat /tmp/tmp.Tw3OrzyLp1
+ rm /tmp/tmp.8Ce26DgL02 /tmp/tmp.Tw3OrzyLp1
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NxJLxohSP1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3uPaxajvSo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NxJLxohSP1
++ cat /tmp/tmp.3uPaxajvSo
++ rm /tmp/tmp.NxJLxohSP1 /tmp/tmp.3uPaxajvSo
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1 --namespace=backup-storage-tls-1228
++ mktemp
+ local LAST_OUT=/tmp/tmp.lZn7adK1DP
++ mktemp
+ local LAST_ERR=/tmp/tmp.2bJND4zChT
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1 --namespace=backup-storage-tls-1228
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lZn7adK1DP
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster1" modified.
+ cat /tmp/tmp.2bJND4zChT
+ rm /tmp/tmp.lZn7adK1DP /tmp/tmp.2bJND4zChT
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.VCn7XjZMki
++ mktemp
+ local LAST_ERR=/tmp/tmp.7eIqjoY1TD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VCn7XjZMki
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.7eIqjoY1TD
+ rm /tmp/tmp.VCn7XjZMki /tmp/tmp.7eIqjoY1TD
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.LWX3MqOrfF
++ mktemp
+ local LAST_ERR=/tmp/tmp.LMMZHj8rnP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.LWX3MqOrfF
namespace/cert-manager created
+ cat /tmp/tmp.LMMZHj8rnP
+ rm /tmp/tmp.LWX3MqOrfF /tmp/tmp.LMMZHj8rnP
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.y1bEZpFdSS
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rmylo3yeHI
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.y1bEZpFdSS
namespace/cert-manager labeled
+ cat /tmp/tmp.Rmylo3yeHI
+ rm /tmp/tmp.y1bEZpFdSS /tmp/tmp.Rmylo3yeHI
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.x79DxnI5tY
++ mktemp
+ local LAST_ERR=/tmp/tmp.adJSLRgUNU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.x79DxnI5tY
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.adJSLRgUNU
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.x79DxnI5tY /tmp/tmp.adJSLRgUNU
+ return 0
+ '[' '' == 4.10 ']'
+ sleep 70
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/issuer.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RzFax8VsLq
++ mktemp
+ local LAST_ERR=/tmp/tmp.XJQrp7Vvh5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/issuer.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RzFax8VsLq
issuer.cert-manager.io/selfsigning-issuer created
+ cat /tmp/tmp.XJQrp7Vvh5
+ rm /tmp/tmp.RzFax8VsLq /tmp/tmp.XJQrp7Vvh5
+ return 0
+ sed s/#namespace/backup-storage-tls-1228/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/cert.yml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.b7BptsdTkL
++ mktemp
+ local LAST_ERR=/tmp/tmp.QKpH6Iv4hJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.b7BptsdTkL
certificate.cert-manager.io/minio-tls-certs created
+ cat /tmp/tmp.QKpH6Iv4hJ
Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.
+ rm /tmp/tmp.b7BptsdTkL /tmp/tmp.QKpH6Iv4hJ
+ return 0
+ sleep 30
+ minio_cert_secret=minio-tls-certs
+ start_minio minio-tls-certs
+ deploy_helm backup-storage-tls-1228
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "hashicorp" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=minio-tls-certs
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n minio-tls-certs ]]
+ endpoint=https://minio-service:9000
+ minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key)
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=minio-tls-certs --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
NAME: minio-service
LAST DEPLOYED: Wed Nov 12 10:34:12 2025
NAMESPACE: backup-storage-tls-1228
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.backup-storage-tls-1228.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace backup-storage-tls-1228 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace backup-storage-tls-1228

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace backup-storage-tls-1228 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace backup-storage-tls-1228 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yc1a7xU9cg
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GY9mOUTv2A
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yc1a7xU9cg
++ cat /tmp/tmp.GY9mOUTv2A
++ rm /tmp/tmp.yc1a7xU9cg /tmp/tmp.GY9mOUTv2A
++ return 0
+ MINIO_POD=minio-service-5f9567876c-n8ft5
+ wait_pod minio-service-5f9567876c-n8ft5
+ local pod=minio-service-5f9567876c-n8ft5
+ local max_retry=480
+ local ns=
++ echo minio-service-5f9567876c-n8ft5
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minio-service-5f9567876c-n8ft5 condition met
waiting for pod/minio-service-5f9567876c-n8ft5 to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.G6i4sVDdDe
++ mktemp
+ local LAST_ERR=/tmp/tmp.gXMDbzG7XB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.G6i4sVDdDe
/usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html
  InsecureRequestWarning)
make_bucket: operator-testing
pod "aws-cli" deleted from backup-storage-tls-1228 namespace
+ cat /tmp/tmp.gXMDbzG7XB
+ rm /tmp/tmp.G6i4sVDdDe /tmp/tmp.gXMDbzG7XB
+ return 0
+ cluster=test-cluster
+ spinup_pxc test-cluster /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/test-cluster.yml
+ local cluster=test-cluster
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/test-cluster.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.3EgYvYCeEK
++ mktemp
+ local LAST_ERR=/tmp/tmp.mYqdQZuW43
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3EgYvYCeEK
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.mYqdQZuW43
+ rm /tmp/tmp.3EgYvYCeEK /tmp/tmp.mYqdQZuW43
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.doTEIy1pyM
++ mktemp
+ local LAST_ERR=/tmp/tmp.PSpDtF6C3C
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.backup-storage-tls-1228~
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.doTEIy1pyM
deployment.apps/pxc-client created
+ cat /tmp/tmp.PSpDtF6C3C
+ rm /tmp/tmp.doTEIy1pyM /tmp/tmp.PSpDtF6C3C
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/test-cluster.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/test-cluster.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.backup-storage-tls-1228~
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/test-cluster.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Upqh9V1Pv2
++ mktemp
+ local LAST_ERR=/tmp/tmp.lYfnkf80AK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Upqh9V1Pv2
perconaxtradbcluster.pxc.percona.com/test-cluster created
+ cat /tmp/tmp.lYfnkf80AK
+ rm /tmp/tmp.Upqh9V1Pv2 /tmp/tmp.lYfnkf80AK
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy test-cluster
++ local target_cluster=test-cluster
+++ kubectl_bin get pxc test-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1Yb8eOl3J7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ivqqbL0WTZ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc test-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.1Yb8eOl3J7
+++ cat /tmp/tmp.ivqqbL0WTZ
+++ rm /tmp/tmp.1Yb8eOl3J7 /tmp/tmp.ivqqbL0WTZ
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc test-cluster -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7GUw8mrK1e
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nrkytahaBU
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc test-cluster -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.7GUw8mrK1e
+++ cat /tmp/tmp.nrkytahaBU
+++ rm /tmp/tmp.7GUw8mrK1e /tmp/tmp.nrkytahaBU
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo test-cluster-proxysql
++ return
+ local proxy=test-cluster-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n backup-storage-tls-1228
++ mktemp
+ local LAST_OUT=/tmp/tmp.YorNM8wJLP
++ mktemp
+ local LAST_ERR=/tmp/tmp.D6BcGpGca4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n backup-storage-tls-1228
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n backup-storage-tls-1228
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n backup-storage-tls-1228
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.YorNM8wJLP
+ cat /tmp/tmp.D6BcGpGca4
error: no matching resources found
+ rm /tmp/tmp.YorNM8wJLP /tmp/tmp.D6BcGpGca4
+ return 1
+ true
+ wait_for_running test-cluster-proxysql 1
+ local name=test-cluster-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod test-cluster-proxysql-0 480
+ local pod=test-cluster-proxysql-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo test-cluster-proxysql-0
+ local container=proxysql
+ set +o xtrace
pod/test-cluster-proxysql-0 condition met
waiting for pod/test-cluster-proxysql-0 to become Ready.Ok
+ wait_for_running test-cluster-pxc 3
+ local name=test-cluster-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod test-cluster-pxc-0 480
+ local pod=test-cluster-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo test-cluster-pxc-0
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/test-cluster-pxc-0 condition met
waiting for pod/test-cluster-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod test-cluster-pxc-1 480
+ local pod=test-cluster-pxc-1
+ local max_retry=480
+ local ns=
++ echo test-cluster-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/test-cluster-pxc-1 condition met
waiting for pod/test-cluster-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod test-cluster-pxc-2 480
+ local pod=test-cluster-pxc-2
+ local max_retry=480
+ local ns=
++ echo test-cluster-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/test-cluster-pxc-2 condition met
waiting for pod/test-cluster-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc test-cluster -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2dIX4WTIKi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8weKtNAQ1v
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2dIX4WTIKi
++ cat /tmp/tmp.8weKtNAQ1v
++ rm /tmp/tmp.2dIX4WTIKi /tmp/tmp.8weKtNAQ1v
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h test-cluster-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h test-cluster-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MacJ4olgWb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1ZdPhg3BwQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MacJ4olgWb
++ cat /tmp/tmp.1ZdPhg3BwQ
++ rm /tmp/tmp.MacJ4olgWb /tmp/tmp.1ZdPhg3BwQ
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h test-cluster-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h test-cluster-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rkROWX5reb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lihgi6O4tP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rkROWX5reb
++ cat /tmp/tmp.lihgi6O4tP
++ rm /tmp/tmp.rkROWX5reb /tmp/tmp.lihgi6O4tP
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-59944c5bbf-lk4f6
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-0.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-0.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-0.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-0.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.RiSwPXfH06
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a0278TKWqn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.RiSwPXfH06
++ cat /tmp/tmp.a0278TKWqn
++ rm /tmp/tmp.RiSwPXfH06 /tmp/tmp.a0278TKWqn
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql /tmp/tmp.nlwoJVsWRC/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-1.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-1.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-1.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-1.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6xXPX4x0rF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DMHp6mQVZn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6xXPX4x0rF
++ cat /tmp/tmp.DMHp6mQVZn
++ rm /tmp/tmp.6xXPX4x0rF /tmp/tmp.DMHp6mQVZn
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql /tmp/tmp.nlwoJVsWRC/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-2.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-2.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h test-cluster-pxc-2.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h test-cluster-pxc-2.test-cluster-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qn6eJbW8Lr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AVhkKSc8Jz
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qn6eJbW8Lr
++ cat /tmp/tmp.AVhkKSc8Jz
++ rm /tmp/tmp.qn6eJbW8Lr /tmp/tmp.AVhkKSc8Jz
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-1.sql /tmp/tmp.nlwoJVsWRC/select-1.sql
++ is_keyring_plugin_in_use test-cluster
++ local cluster=test-cluster
++ kubectl_bin exec -it test-cluster-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.J47EJPJq7L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bTXhIACLsY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it test-cluster-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.J47EJPJq7L
++ cat /tmp/tmp.bTXhIACLsY
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.J47EJPJq7L /tmp/tmp.bTXhIACLsY
++ return 0
+ '[' '' ']'
+ write_test_data test-cluster
+ local cluster=test-cluster
+ local config=
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml
++ get_proxy test-cluster
++ local target_cluster=test-cluster
+++ kubectl_bin get pxc test-cluster -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rCp9gE4Pve
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CJKW6Feu5v
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc test-cluster -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.rCp9gE4Pve
+++ cat /tmp/tmp.CJKW6Feu5v
+++ rm /tmp/tmp.rCp9gE4Pve /tmp/tmp.CJKW6Feu5v
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc test-cluster -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.A1E8FTAeAc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.X0QHzyIiL4
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc test-cluster -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.A1E8FTAeAc
+++ cat /tmp/tmp.X0QHzyIiL4
+++ rm /tmp/tmp.A1E8FTAeAc /tmp/tmp.X0QHzyIiL4
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo test-cluster-proxysql
++ return
+ local proxy=test-cluster-proxysql
+ desc 'write test data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write test data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS test; use test; CREATE TABLE IF NOT EXISTS test (id int PRIMARY KEY) ;' '-h test-cluster-proxysql -uroot -proot_password'
+ local 'command=CREATE DATABASE IF NOT EXISTS test; use test; CREATE TABLE IF NOT EXISTS test (id int PRIMARY KEY) ;'
+ local 'uri=-h test-cluster-proxysql -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YN4tt2Y7B0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jsNbuRt4ui
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.YN4tt2Y7B0
++ cat /tmp/tmp.jsNbuRt4ui
++ rm /tmp/tmp.YN4tt2Y7B0 /tmp/tmp.jsNbuRt4ui
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT test.test (id) VALUES (100500); INSERT test.test (id) VALUES (100501); INSERT test.test (id) VALUES (100502);' '-h test-cluster-proxysql -uroot -proot_password'
+ local 'command=INSERT test.test (id) VALUES (100500); INSERT test.test (id) VALUES (100501); INSERT test.test (id) VALUES (100502);'
+ local 'uri=-h test-cluster-proxysql -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gwWBhmhIEm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.175R6X1enL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gwWBhmhIEm
++ cat /tmp/tmp.175R6X1enL
++ rm /tmp/tmp.gwWBhmhIEm /tmp/tmp.175R6X1enL
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-lk4f6
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-3 'SELECT * from test.test;' '-h test-cluster-pxc-0.test-cluster-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-0.test-cluster-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from test.test;' '-h test-cluster-pxc-0.test-cluster-pxc -uroot -proot_password'
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-0.test-cluster-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wIzBvsZVVV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RiBq2S59wU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wIzBvsZVVV
++ cat /tmp/tmp.RiBq2S59wU
++ rm /tmp/tmp.wIzBvsZVVV /tmp/tmp.RiBq2S59wU
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql /tmp/tmp.nlwoJVsWRC/select-3.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-3 'SELECT * from test.test;' '-h test-cluster-pxc-1.test-cluster-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-1.test-cluster-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from test.test;' '-h test-cluster-pxc-1.test-cluster-pxc -uroot -proot_password'
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-1.test-cluster-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9PYZHEjmx7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q6dZf2scs9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9PYZHEjmx7
++ cat /tmp/tmp.Q6dZf2scs9
++ rm /tmp/tmp.9PYZHEjmx7 /tmp/tmp.Q6dZf2scs9
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql /tmp/tmp.nlwoJVsWRC/select-3.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-3 'SELECT * from test.test;' '-h test-cluster-pxc-2.test-cluster-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-2.test-cluster-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from test.test;' '-h test-cluster-pxc-2.test-cluster-pxc -uroot -proot_password'
+ local 'command=SELECT * from test.test;'
+ local 'uri=-h test-cluster-pxc-2.test-cluster-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AmlpH5PwQh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8zDcKw8gCt
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AmlpH5PwQh
++ cat /tmp/tmp.8zDcKw8gCt
++ rm /tmp/tmp.AmlpH5PwQh /tmp/tmp.8zDcKw8gCt
++ return 0
+ client_pod=pxc-client-59944c5bbf-lk4f6
+ wait_pod pxc-client-59944c5bbf-lk4f6
+ local pod=pxc-client-59944c5bbf-lk4f6
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-lk4f6
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-lk4f6 condition met
waiting for pod/pxc-client-59944c5bbf-lk4f6 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.nlwoJVsWRC/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/compare/select-3.sql /tmp/tmp.nlwoJVsWRC/select-3.sql
++ is_keyring_plugin_in_use test-cluster
++ local cluster=test-cluster
++ egrep -o 'early-plugin-load=keyring_\w+.so'
++ kubectl_bin exec -it test-cluster-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fWoxdiIe5b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DdlT3ww6Ed
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it test-cluster-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fWoxdiIe5b
++ cat /tmp/tmp.DdlT3ww6Ed
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.fWoxdiIe5b /tmp/tmp.DdlT3ww6Ed
++ return 0
+ '[' '' ']'
+ run_backup test-cluster on-demand-backup
+ local cluster=test-cluster
+ local backup=on-demand-backup
+ log 'run pxc-backup/on-demand-backup'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2025-11-12T10:42:23+0000]' run pxc-backup/on-demand-backup
[2025-11-12T10:42:23+0000] run pxc-backup/on-demand-backup
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/on-demand-backup.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.eKPk6UCPMC
++ mktemp
+ local LAST_ERR=/tmp/tmp.RMLIsuPuuR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/backup-storage-tls/conf/on-demand-backup.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eKPk6UCPMC
perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup created
+ cat /tmp/tmp.RMLIsuPuuR
+ rm /tmp/tmp.eKPk6UCPMC /tmp/tmp.RMLIsuPuuR
+ return 0
+ wait_backup on-demand-backup
+ local backup=on-demand-backup
+ local status=Succeeded
+ set +o xtrace
waiting for pxc-backup/on-demand-backup to reach Succeeded state........................................................................................................................................................................................................................................................................................................................................................................2025-11-12T10:31:29.211Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.31.13-gke.1231000"}
2025-11-12T10:31:29.211Z	INFO	setup	Manager starting up	{"gitCommit": "269f3694d9306fde7f2ffd7864a3f22b0ef247c5", "gitBranch": "PR-2234-269f3694", "buildTime": "2025-11-12T09:55:34Z", "goVersion": "go1.25.4", "os": "linux", "arch": "amd64"}
2025-11-12T10:31:29.215Z	INFO	setup	Registering Components.
2025-11-12T10:31:30.241Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2025-11-12T10:31:30.241Z	INFO	setup	Starting the Cmd.
2025-11-12T10:31:30.241Z	INFO	controller-runtime.metrics	Starting metrics server
2025-11-12T10:31:30.241Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2025-11-12T10:31:30.241Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-11-12T10:31:30.241Z	INFO	controller-runtime.webhook	Starting webhook server
2025-11-12T10:31:30.242Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2025-11-12T10:31:30.242Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2025-11-12T10:31:30.242Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2025-11-12T10:31:30.342Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2025-11-12T10:31:30.371Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2025-11-12T10:31:30.371Z	DEBUG	events	percona-xtradb-cluster-operator-b5f9c4897-fdrp8_447de7c8-ae69-4112-90b8-d4c52b052922 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"0e8f4e90-74ad-4ea7-aa39-0ec2f246f316","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1762943490362959009"}, "reason": "LeaderElection"}
2025-11-12T10:31:30.371Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.Secret"}
2025-11-12T10:31:30.371Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2025-11-12T10:31:30.371Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2025-11-12T10:31:30.372Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2025-11-12T10:31:30.472Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2025-11-12T10:31:30.472Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2025-11-12T10:31:30.472Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2025-11-12T10:31:30.472Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2025-11-12T10:31:30.472Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2025-11-12T10:31:30.472Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2025-11-12T10:35:31.009Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e7767635-84e9-4d0b-9ae4-8eaa60f20d43", "version": "1.19.0"}
2025-11-12T10:35:31.443Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.
2025-11-12T10:35:34.482Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.
2025-11-12T10:35:34.505Z	INFO	spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.
2025-11-12T10:35:37.606Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e7767635-84e9-4d0b-9ae4-8eaa60f20d43", "object": "auto-test-cluster-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:35:37.621Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e7767635-84e9-4d0b-9ae4-8eaa60f20d43", "object": "test-cluster-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:35:37.636Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e7767635-84e9-4d0b-9ae4-8eaa60f20d43", "object": "test-cluster-pxc", "kind": "&TypeMeta{Kind:ConfigMap,APIVersion:v1,}"}
2025-11-12T10:35:38.191Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e7767635-84e9-4d0b-9ae4-8eaa60f20d43", "error": "failed to deploy proxysql: updatePod for proxysql: reconcile config: reconcile custom config: create or update config map: configmaps \"test-cluster-pxc\" already exists", "errorVerbose": "configmaps \"test-cluster-pxc\" already exists\ncreate or update config map\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileCustomConfigMap\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:115\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:31\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile custom config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileConfigMaps\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/config.go:33\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:52\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nreconcile config\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).updatePod\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:54\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:577\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nupdatePod for proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:578\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:595\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\nfailed to deploy proxysql\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).deploy\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:596\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:379\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296
2025-11-12T10:35:38.296Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-11-12T10:35:38.332Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"}
2025-11-12T10:35:38.636Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:35:38.729Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:35:38.853Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:35:39.044Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c5f0208e-3ed6-4be3-872b-a7171de10191", "object": "test-cluster-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"}
2025-11-12T10:35:39.895Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "aecaaebb-4681-4b64-991e-cbf907d0f8db", "object": "test-cluster-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-11-12T10:35:39.914Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "aecaaebb-4681-4b64-991e-cbf907d0f8db", "object": "test-cluster-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"}
2025-11-12T10:36:55.682Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab", "user": "operator"}
2025-11-12T10:36:55.724Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab", "user": "monitor"}
2025-11-12T10:36:55.767Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab"}
2025-11-12T10:36:55.798Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab"}
2025-11-12T10:36:55.840Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab", "user": "xtrabackup"}
2025-11-12T10:36:55.877Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab"}
2025-11-12T10:36:55.904Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab", "user": "replication"}
2025-11-12T10:36:55.912Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f71b6483-50ab-4cfd-94a1-8ef11a2c3cab", "err": "get primary pxc pod: not found"}
2025-11-12T10:37:00.684Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8b4aa7cf-b2fe-46af-9a7a-8f3d5bc9269d", "err": "get primary pxc pod: not found"}
2025-11-12T10:37:06.034Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "812e3854-bd10-417a-b856-ea8a291e84ba", "err": "get primary pxc pod: not found"}
2025-11-12T10:37:11.133Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0c5f8462-4b02-4ea7-a8fc-bdbf5e4f11aa", "err": "get primary pxc pod: not found"}
2025-11-12T10:39:15.930Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "914b2611-3a25-4d70-a9c6-aef355b2cb90", "user": "root"}
2025-11-12T10:39:16.065Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "914b2611-3a25-4d70-a9c6-aef355b2cb90", "object": "test-cluster-pitr", "kind": "&TypeMeta{Kind:,APIVersion:,}"}
2025-11-12T10:39:16.149Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "914b2611-3a25-4d70-a9c6-aef355b2cb90", "object": "test-cluster-pitr", "kind": "&TypeMeta{Kind:Deployment,APIVersion:apps/v1,}"}
2025-11-12T10:39:16.214Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "914b2611-3a25-4d70-a9c6-aef355b2cb90", "new version": "8.0.43-34.1"}
2025-11-12T10:39:16.693Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f4caf907-4103-4c30-bc56-c85b376d9de9", "object": "test-cluster-pitr", "kind": "&TypeMeta{Kind:,APIVersion:,}", "hashChanged": false, "metaChanged": true}
  &v1.Service{
  	TypeMeta: v1.TypeMeta{
- 		Kind:       "Service",
+ 		Kind:       "",
- 		APIVersion: "v1",
+ 		APIVersion: "",
  	},
  	ObjectMeta: v1.ObjectMeta{
  		... // 2 identical fields
  		Namespace:                  "backup-storage-tls-1228",
  		SelfLink:                   "",
- 		UID:                        "79c27c60-ca94-47fe-8680-91aa5600688f",
+ 		UID:                        "",
- 		ResourceVersion:            "1762943956144031022",
+ 		ResourceVersion:            "",
  		Generation:                 0,
- 		CreationTimestamp:          v1.Time{Time: s"2025-11-12 10:39:16 +0000 UTC"},
+ 		CreationTimestamp:          v1.Time{},
  		DeletionTimestamp:          nil,
  		DeletionGracePeriodSeconds: nil,
  		Labels:                     nil,
  		Annotations: map[string]string{
- 			"cloud.google.com/neg":         `{"ingress":true}`,
  			"percona.com/last-config-hash": "eyJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJwb3J0Ijo4MDgwLCJ0YXJnZXRQb3J0"...,
  		},
  		OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "test-cluster", UID: "c1802604-3e58-46b9-94ec-9a32cc6e3b7c", ...}},
  		Finalizers:      nil,
- 		ManagedFields: []v1.ManagedFieldsEntry{
- 			{
- 				Manager:    "percona-xtradb-cluster-operator",
- 				Operation:  "Update",
- 				APIVersion: "v1",
- 				Time:       s"2025-11-12 10:39:16 +0000 UTC",
- 				FieldsType: "FieldsV1",
- 				FieldsV1:   s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`...,
- 			},
- 		},
+ 		ManagedFields: nil,
  	},
  	Spec: v1.ServiceSpec{
  		Ports: []v1.ServicePort{
  			{
  				Name:        "http",
- 				Protocol:    "TCP",
+ 				Protocol:    "",
  				AppProtocol: nil,
  				Port:        8080,
  				TargetPort: intstr.IntOrString{
  					Type:   0,
- 					IntVal: 8080,
+ 					IntVal: 0,
  					StrVal: "",
  				},
  				NodePort: 0,
  			},
  		},
  		Selector:                 {"app.kubernetes.io/component": "pitr", "app.kubernetes.io/instance": "test-cluster", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...},
  		ClusterIP:                "34.118.239.154",
- 		ClusterIPs:               []string{"34.118.239.154"},
+ 		ClusterIPs:               nil,
  		Type:                     "ClusterIP",
  		ExternalIPs:              nil,
- 		SessionAffinity:          "None",
+ 		SessionAffinity:          "",
  		LoadBalancerIP:           "",
  		LoadBalancerSourceRanges: nil,
  		... // 3 identical fields
  		PublishNotReadyAddresses:      false,
  		SessionAffinityConfig:         nil,
- 		IPFamilies:                    []v1.IPFamily{"IPv4"},
+ 		IPFamilies:                    nil,
- 		IPFamilyPolicy:                &"SingleStack",
+ 		IPFamilyPolicy:                nil,
  		AllocateLoadBalancerNodePorts: nil,
  		LoadBalancerClass:             nil,
- 		InternalTrafficPolicy:         &"Cluster",
+ 		InternalTrafficPolicy:         nil,
  		TrafficDistribution:           nil,
  	},
  	Status: {},
  }

2025-11-12T10:39:17.954Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "914b2611-3a25-4d70-a9c6-aef355b2cb90"}
2025-11-12T10:39:22.270Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "33cba102-2e62-4a53-8e63-63695bb61d71"}
2025-11-12T10:39:27.798Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a7fc193a-0a6e-4523-87cd-ccd2cf44f088"}
2025-11-12T10:39:32.867Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f173da98-e8e7-4711-aa6a-2213c453e8fb"}
2025-11-12T10:39:38.656Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "839699b4-71d6-4025-abcd-241dccad3edb"}
2025-11-12T10:39:43.765Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "4e65bf35-e8b5-4ae9-b5f8-5d4398b1bfd7"}
2025-11-12T10:39:48.751Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "352c118a-3850-4d43-a7cf-3680a54161f7"}
2025-11-12T10:39:54.667Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f18a2de9-f0bf-4102-87b3-77a0b8934472"}
2025-11-12T10:39:59.554Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "695c9a29-49a0-415b-9446-5bd45240f5f9"}
2025-11-12T10:40:04.851Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0af27e75-dd0f-468f-940e-0c364b59aa58"}
2025-11-12T10:40:10.373Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "6218f00e-22dc-4781-ac2e-a7a515f4f22c"}
2025-11-12T10:40:15.803Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "41453cd7-7533-4d5a-b4b7-82f69f4f6a1c"}
2025-11-12T10:40:21.053Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b492e9c5-d853-4d05-85e4-9a2801f98b05"}
2025-11-12T10:40:26.389Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "60f66b70-bcf4-4807-9517-87a61df8f8af"}
2025-11-12T10:40:31.584Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7cd7ed9f-5c36-45e5-9828-b0ec9f757111"}
2025-11-12T10:40:36.756Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "061b3e65-72af-4002-8c1d-7e03a63d1793"}
2025-11-12T10:40:41.984Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b0002532-0028-4d30-a9f9-31e4d0821db0"}
2025-11-12T10:40:47.463Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "04504431-73f6-470f-95e6-fa8b7b2406f2"}
2025-11-12T10:40:52.695Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "bbcdef08-2cbc-42b0-bb5b-2e474bdb1471"}
2025-11-12T10:40:57.869Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "339f10de-5b7d-46a2-a491-871e8d1b59f5"}
2025-11-12T10:41:02.981Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "65d6f432-18b5-47bf-8b63-23ff1f263a9d"}
2025-11-12T10:41:08.625Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7131870e-e2f4-4dd6-953a-ef53647d0afb"}
2025-11-12T10:41:13.784Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "6a0caf67-dfdc-4e29-be7f-26bfe647c50d"}
2025-11-12T10:41:19.093Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0908c66f-b351-4b3d-b190-c51f3e66b7ed"}
2025-11-12T10:41:24.257Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f945e0a6-89c0-469a-910e-1298e43ee33a"}
2025-11-12T10:41:29.683Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7449aa57-95cb-4281-abda-ff4041ff3292"}
2025-11-12T10:41:35.054Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "caea30c7-85ab-4326-894a-b391453aa115"}
2025-11-12T10:41:40.306Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f6f900da-2572-4646-bc8a-36332e76f068"}
2025-11-12T10:41:45.809Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "49c2d7e0-9cd4-4d75-aa61-835c82059edb"}
2025-11-12T10:41:50.793Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5af951ed-c3c4-448a-8695-8173a80b7bbe"}
2025-11-12T10:41:56.454Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0acb2ed2-ffd4-4944-a36f-018a40e46c9d"}
2025-11-12T10:42:01.198Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "43f9e5db-3c2a-4ee2-bec5-bb0b9cf51d14"}
2025-11-12T10:42:06.661Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a7849a37-7a73-41c7-9268-7b79724a371f"}
2025-11-12T10:42:12.184Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "ad39a5b5-eb2e-418e-b799-d62fba26b991"}
2025-11-12T10:42:17.200Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2f32872e-64af-4d0a-b235-360be1352983"}
2025-11-12T10:42:22.772Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "73ac85fa-c007-4355-beae-3f4a279fb1fe"}
2025-11-12T10:42:25.989Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "namespace": "backup-storage-tls-1228", "name": "on-demand-backup", "reconcileID": "7b88e253-052c-40c7-827c-81359fd6241a", "cluster": "test-cluster", "storage": "minio", "allowed": false}
2025-11-12T10:42:26.155Z	INFO	Created a new backup job	{"controller": "pxcbackup-controller", "namespace": "backup-storage-tls-1228", "name": "on-demand-backup", "reconcileID": "7b88e253-052c-40c7-827c-81359fd6241a", "namespace": "backup-storage-tls-1228", "name": "xb-on-demand-backup"}
2025-11-12T10:42:26.156Z	DEBUG	Check if parallel backups are allowed	{"controller": "pxcbackup-controller", "namespace": "backup-storage-tls-1228", "name": "on-demand-backup", "reconcileID": "47baed12-3d4b-41d9-a941-3f1d9ffcc448", "cluster": "test-cluster", "storage": "minio", "allowed": false}
2025-11-12T10:42:26.235Z	INFO	Releasing backup lock	{"controller": "pxcbackup-controller", "namespace": "backup-storage-tls-1228", "name": "on-demand-backup", "reconcileID": "47baed12-3d4b-41d9-a941-3f1d9ffcc448", "cluster": "test-cluster", "storage": "minio", "lease": "pxc-test-cluster-backup-lock"}
2025-11-12T10:42:26.303Z	ERROR	Reconciler error	{"controller": "pxcbackup-controller", "namespace": "backup-storage-tls-1228", "name": "on-demand-backup", "reconcileID": "47baed12-3d4b-41d9-a941-3f1d9ffcc448", "error": "create backup job: create backup job: jobs.batch \"xb-on-demand-backup\" already exists", "errorVerbose": "jobs.batch \"xb-on-demand-backup\" already exists\ncreate backup job\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).createBackupJob\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:383\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:291\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693\ncreate backup job\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup.(*ReconcilePerconaXtraDBClusterBackup).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxcbackup/controller.go:293\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1693"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296
2025-11-12T10:42:27.972Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "de28e3ed-7fdb-43ae-a985-179071b66695"}
2025-11-12T10:42:33.377Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8f905481-4e3e-48d7-874e-f3eef6ff2901"}
2025-11-12T10:42:38.677Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7dc1a09a-e807-4a15-8300-98085a9203b6"}
2025-11-12T10:42:43.899Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "67a5fe87-5754-4eb4-803f-f40089ffc58a"}
2025-11-12T10:42:49.208Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "17fba9fc-98e6-4ef5-b8f4-f82ceabab043"}
2025-11-12T10:42:54.553Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "15e99e48-6a7e-4644-8173-ec8a8a8d58cb"}
2025-11-12T10:42:59.684Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a0cdfe93-6bfc-4651-a267-5b333f535a8f"}
2025-11-12T10:43:04.911Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "176d6750-2891-4a04-b743-1fb3c2251aa1"}
2025-11-12T10:43:10.495Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "daed21d2-4473-4f47-b92e-1c4d7080895f"}
2025-11-12T10:43:16.064Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "296d40a1-fa28-4af9-9714-5a162d97ec79"}
2025-11-12T10:43:21.289Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b50d99b4-0958-4821-a02b-ad80586af481"}
2025-11-12T10:43:26.650Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a1319dd5-dd5d-4303-865c-16820763851e"}
2025-11-12T10:43:31.584Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0f03e2b4-baa6-4aa9-b54b-528b9045e22d"}
2025-11-12T10:43:37.154Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "487b6b02-65ab-4a12-aae2-ef30f1f97496"}
2025-11-12T10:43:42.258Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "48091f58-71d4-42b3-aa82-ed4d119bb321"}
2025-11-12T10:43:47.896Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b4fdf8d1-b91c-4a4d-90bc-190dde2285b8"}
2025-11-12T10:43:53.593Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "1ecf3167-5db5-4866-bdc1-cf2d186c3384"}
2025-11-12T10:43:58.764Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "57a6a227-8902-4414-8f39-25b65fbf4c1b"}
2025-11-12T10:44:04.000Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7de0ca2c-b3bb-4395-a869-776e945155c7"}
2025-11-12T10:44:09.499Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "70451ff4-2410-43c0-badf-821857188a41"}
2025-11-12T10:44:14.485Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "9e7c2c38-c633-46ff-b740-238fdc90aeb9"}
2025-11-12T10:44:19.677Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5d59cdec-7e89-4af3-bc4b-18db8009633b"}
2025-11-12T10:44:24.577Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b0f08412-5023-4b2a-bec9-c2c255025073"}
2025-11-12T10:44:30.253Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c758a370-50d5-47d6-ae41-c9c6b6d4e3e7"}
2025-11-12T10:44:35.483Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5845e32d-413c-4264-a607-400ac9755938"}
2025-11-12T10:44:40.692Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f81036ef-bde3-43d4-8b74-1d3b9c418c93"}
2025-11-12T10:44:45.894Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "1d8130e8-fa00-44aa-9cee-8d1e100725b6"}
2025-11-12T10:44:51.406Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5bf69cc8-227f-4624-9d50-645bdb695ddd"}
2025-11-12T10:44:56.608Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8ef957e5-b3bb-4ce0-b2ca-586155c1729a"}
2025-11-12T10:45:01.956Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "259040f7-5f1a-4389-a2bf-727f108c058f"}
2025-11-12T10:45:07.088Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "003c54e9-3fa5-4d27-8f63-65e500ce2f8f"}
2025-11-12T10:45:12.406Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a10b84d1-499f-4253-9f27-775e9c7f33dc"}
2025-11-12T10:45:17.599Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "603966d5-bb1c-4d9b-b069-33b367295127"}
2025-11-12T10:45:23.075Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "39cdcf1a-c607-4685-9f88-1a350480130d"}
2025-11-12T10:45:28.308Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0fefa23e-4930-49e7-9128-b1a40aed776c"}
2025-11-12T10:45:33.694Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "bd687b92-62d1-4c5a-982e-7af00d4e9f11"}
2025-11-12T10:45:38.963Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7bb655d8-28ce-4371-b965-5d81ef1c729a"}
2025-11-12T10:45:44.074Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "3d78502a-8610-4a8a-a6b1-2e95717e5aa6"}
2025-11-12T10:45:49.482Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "001b476a-62eb-4805-9c12-de3f0153c417"}
2025-11-12T10:45:54.697Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2617e01f-a9f0-4892-b408-e6b5edb1dbd8"}
2025-11-12T10:46:00.078Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "69757ace-1a76-4547-a253-f1a508c5c819"}
2025-11-12T10:46:05.210Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "dc946d63-9ca8-4b03-89b6-e6f68e300073"}
2025-11-12T10:46:10.298Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "68a7f5e4-b272-41f8-b66c-db527ee07716"}
2025-11-12T10:46:15.912Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f667ef29-f2a3-4225-9e4c-5c109875ba6e"}
2025-11-12T10:46:20.959Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2725278e-8f1e-4944-9bca-14b76fae8e05"}
2025-11-12T10:46:26.181Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "00c186a5-58ae-4fa1-af0b-9562129505e6"}
2025-11-12T10:46:31.657Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "fe11e960-7014-427c-aa1d-7a459f010ddc"}
2025-11-12T10:46:36.981Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a3e70bd6-a677-417e-8d5a-2d8a3f7e8d68"}
2025-11-12T10:46:42.282Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2c60cb60-7921-4814-ac22-70246a878e53"}
2025-11-12T10:46:47.750Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f659f114-ff9c-4eb4-94df-ef1ba8b4d58d"}
2025-11-12T10:46:52.957Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e5d441eb-565f-4f1a-a0ca-ebddb35baac5"}
2025-11-12T10:46:56.311Z	INFO	controller-runtime.cache	Warning: watch ended with error	{"reflector": "pkg/mod/k8s.io/client-go@v0.34.1/tools/cache/reflector.go:290", "type": "*v1.Event", "err": "very short watch: pkg/mod/k8s.io/client-go@v0.34.1/tools/cache/reflector.go:290: Unexpected watch close - watch lasted less than a second and no items received"}
2025-11-12T10:46:58.362Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "ade9665a-3a60-4ec7-9ea8-3a1ebf49b68d"}
2025-11-12T10:47:03.375Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8f4eb228-8704-482d-af72-a212aad0045a"}
2025-11-12T10:47:08.602Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e00baac2-fa0b-4326-919d-af2542589271"}
2025-11-12T10:47:13.906Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "ff77a0a9-f988-4a28-a82d-72215ad415c2"}
2025-11-12T10:47:19.351Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "fe9ee56a-2655-45d6-8b9b-7f1a377fd4ef"}
2025-11-12T10:47:24.601Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "4400745b-944c-4567-8de3-77b1312d746d"}
2025-11-12T10:47:29.692Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e791d472-197a-464a-b0c4-ac085e00452d"}
2025-11-12T10:47:35.086Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2344f7cc-dbea-408d-a0af-067dfc84940a"}
2025-11-12T10:47:40.753Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5d360d41-58ef-4582-8a07-7b119bd38d6c"}
2025-11-12T10:47:45.723Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "12e8a589-a3af-493b-a1f2-4608d72a9a88"}
2025-11-12T10:47:51.167Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0dd6eaf1-2ff3-4f31-9d0c-fa070ba863f7"}
2025-11-12T10:47:56.652Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "646e800c-3e60-46e1-9575-9acf86640f2b"}
2025-11-12T10:48:01.693Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "b69fb910-6ccc-4a98-b416-5e0fdd64c6e1"}
2025-11-12T10:48:07.177Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "97e2d735-a7f2-428a-902f-b94b346f9a3b"}
2025-11-12T10:48:12.179Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "cbd59613-4575-4ebc-9b6d-87cea5436227"}
2025-11-12T10:48:17.798Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "64f6f5b7-1c11-478d-8302-5405621f989d"}
2025-11-12T10:48:22.987Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "fc8117a5-aaf3-42cd-ba42-8380b8353f81"}
2025-11-12T10:48:28.262Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "67975c4b-93bb-4223-aa8e-7226e99f4941"}
2025-11-12T10:48:33.508Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c76cb36c-e858-4941-a25f-231492c7aea3"}
2025-11-12T10:48:38.881Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "6d540388-6283-4288-bb6c-9e6ce6f817c4"}
2025-11-12T10:48:44.069Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5ff57a78-a299-4ce1-b31e-9e271f77b9ec"}
2025-11-12T10:48:49.359Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "37a77fc7-6886-4613-ab9e-7dbe2e26295f"}
2025-11-12T10:48:54.603Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "d975dbe8-8ca6-4525-9fe9-7cfd38e6f8ba"}
2025-11-12T10:48:59.971Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f57f0821-04bd-4915-9f83-2528f1f97ae0"}
2025-11-12T10:49:05.089Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "1754a219-46e5-4d88-be3a-44ac43a033cd"}
2025-11-12T10:49:10.593Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e13b82a9-2248-4baf-9f1e-e377690b8d28"}
2025-11-12T10:49:15.876Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "3be4da76-c0cd-4d72-a493-b830adbf6097"}
2025-11-12T10:49:20.785Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8bfad70c-1ac2-4426-bb39-c0540cc61451"}
2025-11-12T10:49:26.289Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "9bec0b6e-66f6-49ac-afa3-58299687f368"}
2025-11-12T10:49:31.678Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "d87fdbf3-e5d9-42fb-9629-078feb7923ff"}
2025-11-12T10:49:37.053Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a45262e1-bb35-414a-a8af-67195d5b7ac2"}
2025-11-12T10:49:42.180Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "8405d90a-950d-41dd-a5f5-049730d5efff"}
2025-11-12T10:49:47.478Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "af7f7555-d72c-424d-9cf8-600f4b4d63d0"}
2025-11-12T10:49:52.955Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "6884c8ec-a35c-4bdf-9192-2768ff79a970"}
2025-11-12T10:49:58.179Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "1f7ed134-3da2-4740-bd9a-2c8ffb324307"}
2025-11-12T10:50:03.473Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "728af60d-b5c7-4fb4-ab7f-9893ba3376b4"}
2025-11-12T10:50:08.589Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f45b18f5-c10c-49f9-ab38-9324616d243e"}
2025-11-12T10:50:13.764Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c31f34b7-fe9e-4faa-ba58-1094b40a945a"}
2025-11-12T10:50:19.302Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "cb4f4132-b252-4daa-8b34-1728cbc38707"}
2025-11-12T10:50:24.485Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "d8417691-4cf9-4c8e-83ac-21b3f29b3e35"}
2025-11-12T10:50:29.776Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "17a0204a-a61c-4dfe-831b-fd509dce54d8"}
2025-11-12T10:50:34.763Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "048f71f3-ba22-4399-ab34-0730651b04cf"}
2025-11-12T10:50:40.461Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "aafbea08-7454-4f40-abac-e9af0af5b040"}
2025-11-12T10:50:45.671Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "36defe59-4b8f-42ea-99cd-40c550f3a56d"}
2025-11-12T10:50:50.982Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "a453f4e3-4a50-4a3c-a8da-baeba428cb14"}
2025-11-12T10:50:56.356Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "0d91e0de-86c2-4f20-8eb1-164e34a2c5eb"}
2025-11-12T10:51:01.852Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "c943e922-4473-42b2-97c1-551c2a4cd366"}
2025-11-12T10:51:06.958Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "f6f31ff3-8a53-4a4a-980f-2db52278b7a3"}
2025-11-12T10:51:12.083Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "bf6da7b9-210d-445b-ac06-2b33cad2a103"}
2025-11-12T10:51:17.776Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "ee6cde02-6032-418f-a521-7efe34418d26"}
2025-11-12T10:51:23.090Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "4150021b-843e-44a5-8088-4d140fd218b3"}
2025-11-12T10:51:28.279Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "7597ec8a-1d2b-499f-a302-e944bdcd83ee"}
2025-11-12T10:51:33.675Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "feb4124f-069d-4265-8a1f-b2c5992dae5a"}
2025-11-12T10:51:38.757Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "3d43b911-4382-4d61-bd50-18a74b956e81"}
2025-11-12T10:51:44.054Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "919eb3e9-f937-40be-9f19-13d58898c508"}
2025-11-12T10:51:49.553Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "3b4ffcaa-bb0f-44f4-95a6-bdddc69b551c"}
2025-11-12T10:51:54.694Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "2caf1840-8813-476b-8994-427ecb02eaab"}
2025-11-12T10:51:59.688Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "e2df327d-e49b-4b75-9701-3f9ef483afff"}
2025-11-12T10:52:05.209Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "backup-storage-tls-1228", "name": "test-cluster", "reconcileID": "5bd07ae0-e3dd-4d56-a47f-c13f23571a4c"}
max retry count 360 reached. something went wrong with operator or kubernetes cluster