Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/logs/demand-backup-cloud-8-0.log Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 + main + '[' -n '' ']' + create_infra demand-backup-cloud-27138 + local ns=demand-backup-cloud-27138 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + grep -v NAMESPACE + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + kubectl patch pxc -n demand-backup-cloud-15248 demand-backup-cloud --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/demand-backup-cloud patched + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.LuHPBlkRcN ++ mktemp + local LAST_ERR=/tmp/tmp.247fXZBFNi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LuHPBlkRcN perconaxtradbcluster.pxc.percona.com "demand-backup-cloud" deleted from demand-backup-cloud-15248 namespace + cat /tmp/tmp.247fXZBFNi + rm /tmp/tmp.LuHPBlkRcN /tmp/tmp.247fXZBFNi + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.FByzFHwkQ1 ++ mktemp + local LAST_ERR=/tmp/tmp.tl82ozDgmr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FByzFHwkQ1 perconaxtradbclusterbackup.pxc.percona.com "on-demand-backup-aws-s3" deleted from demand-backup-cloud-15248 namespace + cat /tmp/tmp.tl82ozDgmr + rm /tmp/tmp.FByzFHwkQ1 /tmp/tmp.tl82ozDgmr + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.Rbnq3BsJ2A ++ mktemp + local LAST_ERR=/tmp/tmp.keOJMLI7h2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Rbnq3BsJ2A perconaxtradbclusterrestore.pxc.percona.com "on-demand-backup-aws-s3" deleted from demand-backup-cloud-15248 namespace + cat /tmp/tmp.keOJMLI7h2 + rm /tmp/tmp.Rbnq3BsJ2A /tmp/tmp.keOJMLI7h2 + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ awk '{print $1}' ++ grep chaos-mesh.org + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.cwcVCyi7YM ++ mktemp + local LAST_ERR=/tmp/tmp.sOFehzN6J9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.luERd0DUXO ++ mktemp + local LAST_ERR=/tmp/tmp.HS9JCMGDHU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cwcVCyi7YM + cat /tmp/tmp.sOFehzN6J9 + rm /tmp/tmp.cwcVCyi7YM /tmp/tmp.sOFehzN6J9 + return 0 namespace "demand-backup-cloud-15248" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.luERd0DUXO namespace "pxc-operator" deleted + cat /tmp/tmp.HS9JCMGDHU + rm /tmp/tmp.luERd0DUXO /tmp/tmp.HS9JCMGDHU + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.AlZQEUhF57 ++ mktemp + local LAST_ERR=/tmp/tmp.Lxp03zfavD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AlZQEUhF57 namespace/pxc-operator created + cat /tmp/tmp.Lxp03zfavD + rm /tmp/tmp.AlZQEUhF57 /tmp/tmp.Lxp03zfavD + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.qH0HVc8jED +++ mktemp ++ local LAST_ERR=/tmp/tmp.Btz1freyL2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qH0HVc8jED ++ cat /tmp/tmp.Btz1freyL2 ++ rm /tmp/tmp.qH0HVc8jED /tmp/tmp.Btz1freyL2 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.btFPZ08Z4l ++ mktemp + local LAST_ERR=/tmp/tmp.sXxbJ4YDqX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.btFPZ08Z4l Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3" modified. + cat /tmp/tmp.sXxbJ4YDqX + rm /tmp/tmp.btFPZ08Z4l /tmp/tmp.sXxbJ4YDqX + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rCfma9Dkme ++ mktemp + local LAST_ERR=/tmp/tmp.XN7esTqSPq + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rCfma9Dkme customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.XN7esTqSPq + rm /tmp/tmp.rCfma9Dkme /tmp/tmp.XN7esTqSPq + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-rbac.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.51rwGmuXEW + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_ERR=/tmp/tmp.85PlEctB6b + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.51rwGmuXEW clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.85PlEctB6b + rm /tmp/tmp.51rwGmuXEW /tmp/tmp.85PlEctB6b + return 0 + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ mktemp + local LAST_OUT=/tmp/tmp.y0piZNKyWT ++ mktemp + local LAST_ERR=/tmp/tmp.7Ycvam3ogv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.y0piZNKyWT deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.7Ycvam3ogv + rm /tmp/tmp.y0piZNKyWT /tmp/tmp.7Ycvam3ogv + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.hTTUWRKaoc ++ mktemp + local LAST_ERR=/tmp/tmp.3dxfbBi0EP + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hTTUWRKaoc pod/percona-xtradb-cluster-operator-b5f9c4897-9lg6t condition met + cat /tmp/tmp.3dxfbBi0EP + rm /tmp/tmp.hTTUWRKaoc /tmp/tmp.3dxfbBi0EP + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ykpf0WgDBr +++ mktemp ++ local LAST_ERR=/tmp/tmp.K9vFMFEHA2 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Ykpf0WgDBr ++ cat /tmp/tmp.K9vFMFEHA2 ++ rm /tmp/tmp.Ykpf0WgDBr /tmp/tmp.K9vFMFEHA2 ++ return 0 + wait_pod percona-xtradb-cluster-operator-b5f9c4897-9lg6t 480 pxc-operator + local pod=percona-xtradb-cluster-operator-b5f9c4897-9lg6t + local max_retry=480 + local ns=pxc-operator ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-b5f9c4897-9lg6t + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-b5f9c4897-9lg6t condition met waiting for pod/percona-xtradb-cluster-operator-b5f9c4897-9lg6t to become Ready.Ok + sleep 3 + create_namespace demand-backup-cloud-27138 + local namespace=demand-backup-cloud-27138 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces demand-backup-cloud-27138' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces demand-backup-cloud-27138 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace demand-backup-cloud-27138 + xargs kubectl delete ns + kubectl_bin get ns + awk '{print$1}' ++ mktemp ++ mktemp + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.5lxc9NB6Z2 + local LAST_OUT=/tmp/tmp.7TbfOq1U0o ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.5zYGf3rkKO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.iYXkwF6W2H + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-cloud-27138 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-cloud-27138 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5lxc9NB6Z2 + cat /tmp/tmp.5zYGf3rkKO + rm /tmp/tmp.5lxc9NB6Z2 /tmp/tmp.5zYGf3rkKO + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace demand-backup-cloud-27138 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.7TbfOq1U0o + cat /tmp/tmp.iYXkwF6W2H Error from server (NotFound): namespaces "demand-backup-cloud-27138" not found + rm /tmp/tmp.7TbfOq1U0o /tmp/tmp.iYXkwF6W2H + return 1 + : + wait_for_delete namespace/demand-backup-cloud-27138 + local res=namespace/demand-backup-cloud-27138 + echo -n 'waiting for namespace/demand-backup-cloud-27138 to be deleted' waiting for namespace/demand-backup-cloud-27138 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "demand-backup-cloud-27138" not found + desc 'create namespace demand-backup-cloud-27138' + set +o xtrace ----------------------------------------------------------------------------------- create namespace demand-backup-cloud-27138 ----------------------------------------------------------------------------------- + kubectl_bin create namespace demand-backup-cloud-27138 ++ mktemp + local LAST_OUT=/tmp/tmp.k6hU7khx2C ++ mktemp + local LAST_ERR=/tmp/tmp.6ogH5CGGGU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace demand-backup-cloud-27138 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k6hU7khx2C namespace/demand-backup-cloud-27138 created + cat /tmp/tmp.6ogH5CGGGU + rm /tmp/tmp.k6hU7khx2C /tmp/tmp.6ogH5CGGGU + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.vUd253w1w9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JbQ0yuxwRX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vUd253w1w9 ++ cat /tmp/tmp.JbQ0yuxwRX ++ rm /tmp/tmp.vUd253w1w9 /tmp/tmp.JbQ0yuxwRX ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3 --namespace=demand-backup-cloud-27138 ++ mktemp + local LAST_OUT=/tmp/tmp.6R1Yn9pkKP ++ mktemp + local LAST_ERR=/tmp/tmp.r3cOQzcBVd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3 --namespace=demand-backup-cloud-27138 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6R1Yn9pkKP Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2234-269f3694-3-cluster3" modified. + cat /tmp/tmp.r3cOQzcBVd + rm /tmp/tmp.6R1Yn9pkKP /tmp/tmp.r3cOQzcBVd + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.I8E6GAPQYV ++ mktemp + local LAST_ERR=/tmp/tmp.4qi5m05pf1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.I8E6GAPQYV secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.4qi5m05pf1 + rm /tmp/tmp.I8E6GAPQYV /tmp/tmp.4qi5m05pf1 + return 0 + cluster=demand-backup-cloud + spinup_pxc demand-backup-cloud /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml + local cluster=demand-backup-cloud + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.jChyVexdtg ++ mktemp + local LAST_ERR=/tmp/tmp.uZJ3HkxXeb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.jChyVexdtg secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.uZJ3HkxXeb + rm /tmp/tmp.jChyVexdtg /tmp/tmp.uZJ3HkxXeb + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-cloud-27138~ + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.d79lM4sB86 ++ mktemp + local LAST_ERR=/tmp/tmp.NNJdfdzllM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.d79lM4sB86 deployment.apps/pxc-client created + cat /tmp/tmp.NNJdfdzllM + rm /tmp/tmp.d79lM4sB86 /tmp/tmp.NNJdfdzllM + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2234-269f3694#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-cloud-27138~ + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/demand-backup-cloud.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.Zes8P4hs2M ++ mktemp + local LAST_ERR=/tmp/tmp.HODaOgTKCb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Zes8P4hs2M perconaxtradbcluster.pxc.percona.com/demand-backup-cloud created + cat /tmp/tmp.HODaOgTKCb + rm /tmp/tmp.Zes8P4hs2M /tmp/tmp.HODaOgTKCb + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy demand-backup-cloud ++ local target_cluster=demand-backup-cloud +++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H8AaGSDBfr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AkqVWAFDNU +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.H8AaGSDBfr +++ cat /tmp/tmp.AkqVWAFDNU +++ rm /tmp/tmp.H8AaGSDBfr /tmp/tmp.AkqVWAFDNU +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ybh2m6NvrA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DOxlYmhvwA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Ybh2m6NvrA +++ cat /tmp/tmp.DOxlYmhvwA +++ rm /tmp/tmp.Ybh2m6NvrA /tmp/tmp.DOxlYmhvwA +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo demand-backup-cloud-proxysql ++ return + local proxy=demand-backup-cloud-proxysql + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-27138 ++ mktemp + local LAST_OUT=/tmp/tmp.vALPGuYoBd ++ mktemp + local LAST_ERR=/tmp/tmp.uqzFqrSW9F + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-27138 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-27138 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n demand-backup-cloud-27138 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.vALPGuYoBd + cat /tmp/tmp.uqzFqrSW9F error: no matching resources found + rm /tmp/tmp.vALPGuYoBd /tmp/tmp.uqzFqrSW9F + return 1 + true + wait_for_running demand-backup-cloud-proxysql 1 + local name=demand-backup-cloud-proxysql + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod demand-backup-cloud-proxysql-0 480 + local pod=demand-backup-cloud-proxysql-0 + local max_retry=480 + local ns= ++ echo demand-backup-cloud-proxysql-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/demand-backup-cloud-proxysql-0 condition met waiting for pod/demand-backup-cloud-proxysql-0 to become Ready.Ok + wait_for_running demand-backup-cloud-pxc 3 + local name=demand-backup-cloud-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod demand-backup-cloud-pxc-0 480 + local pod=demand-backup-cloud-pxc-0 + local max_retry=480 + local ns= ++ echo demand-backup-cloud-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/demand-backup-cloud-pxc-0 condition met waiting for pod/demand-backup-cloud-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod demand-backup-cloud-pxc-1 480 + local pod=demand-backup-cloud-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo demand-backup-cloud-pxc-1 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/demand-backup-cloud-pxc-1 condition met waiting for pod/demand-backup-cloud-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod demand-backup-cloud-pxc-2 480 + local pod=demand-backup-cloud-pxc-2 + local max_retry=480 + local ns= ++ echo demand-backup-cloud-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/demand-backup-cloud-pxc-2 condition met waiting for pod/demand-backup-cloud-pxc-2 to become Ready.Ok + sleep 10 ++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.dmkH9ujMlp +++ mktemp ++ local LAST_ERR=/tmp/tmp.HINSqvUfQ7 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dmkH9ujMlp ++ cat /tmp/tmp.HINSqvUfQ7 ++ rm /tmp/tmp.dmkH9ujMlp /tmp/tmp.HINSqvUfQ7 ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4bw8d2NZmf +++ mktemp ++ local LAST_ERR=/tmp/tmp.S0KDla9pL4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4bw8d2NZmf ++ cat /tmp/tmp.S0KDla9pL4 ++ rm /tmp/tmp.4bw8d2NZmf /tmp/tmp.S0KDla9pL4 ++ return 0 + client_pod=pxc-client-59944c5bbf-j278g + wait_pod pxc-client-59944c5bbf-j278g + local pod=pxc-client-59944c5bbf-j278g + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-59944c5bbf-j278g + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j278g condition met waiting for pod/pxc-client-59944c5bbf-j278g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h demand-backup-cloud-proxysql -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p9RJiqTe35 +++ mktemp ++ local LAST_ERR=/tmp/tmp.GizaMX2fFe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p9RJiqTe35 ++ cat /tmp/tmp.GizaMX2fFe ++ rm /tmp/tmp.p9RJiqTe35 /tmp/tmp.GizaMX2fFe ++ return 0 + client_pod=pxc-client-59944c5bbf-j278g + wait_pod pxc-client-59944c5bbf-j278g + local pod=pxc-client-59944c5bbf-j278g + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j278g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j278g condition met waiting for pod/pxc-client-59944c5bbf-j278g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-0.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JOZwnD9zPi +++ mktemp ++ local LAST_ERR=/tmp/tmp.3D6JPI90ht ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JOZwnD9zPi ++ cat /tmp/tmp.3D6JPI90ht ++ rm /tmp/tmp.JOZwnD9zPi /tmp/tmp.3D6JPI90ht ++ return 0 + client_pod=pxc-client-59944c5bbf-j278g + wait_pod pxc-client-59944c5bbf-j278g + local pod=pxc-client-59944c5bbf-j278g + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j278g ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j278g condition met waiting for pod/pxc-client-59944c5bbf-j278g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.EzWduxsHWL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.EzWduxsHWL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-1.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HFJJFNoC7n +++ mktemp ++ local LAST_ERR=/tmp/tmp.uiloH1x5NH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HFJJFNoC7n ++ cat /tmp/tmp.uiloH1x5NH ++ rm /tmp/tmp.HFJJFNoC7n /tmp/tmp.uiloH1x5NH ++ return 0 + client_pod=pxc-client-59944c5bbf-j278g + wait_pod pxc-client-59944c5bbf-j278g + local pod=pxc-client-59944c5bbf-j278g + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j278g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j278g condition met waiting for pod/pxc-client-59944c5bbf-j278g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.EzWduxsHWL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.EzWduxsHWL/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h demand-backup-cloud-pxc-2.demand-backup-cloud-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eRNzpc9Y9e +++ mktemp ++ local LAST_ERR=/tmp/tmp.4sQvmoCmcC ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eRNzpc9Y9e ++ cat /tmp/tmp.4sQvmoCmcC ++ rm /tmp/tmp.eRNzpc9Y9e /tmp/tmp.4sQvmoCmcC ++ return 0 + client_pod=pxc-client-59944c5bbf-j278g + wait_pod pxc-client-59944c5bbf-j278g + local pod=pxc-client-59944c5bbf-j278g + local max_retry=480 + local ns= ++ echo pxc-client-59944c5bbf-j278g ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-59944c5bbf-j278g condition met waiting for pod/pxc-client-59944c5bbf-j278g to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + '[' '!' -s /tmp/tmp.EzWduxsHWL/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/compare/select-1.sql /tmp/tmp.EzWduxsHWL/select-1.sql ++ is_keyring_plugin_in_use demand-backup-cloud ++ local cluster=demand-backup-cloud ++ kubectl_bin exec -it demand-backup-cloud-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Vo791WEMbX +++ mktemp ++ local LAST_ERR=/tmp/tmp.QQ55JisBuG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it demand-backup-cloud-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Vo791WEMbX ++ cat /tmp/tmp.QQ55JisBuG Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.Vo791WEMbX /tmp/tmp.QQ55JisBuG ++ return 0 + '[' '' ']' + backup_name_aws=on-demand-backup-aws-s3 + backup_name_gcp=on-demand-backup-gcp-cs + backup_name_azure=on-demand-backup-azure-blob + desc 'Run backup on-demand-backup-aws-s3 for demand-backup-cloud cluster' + set +o xtrace ----------------------------------------------------------------------------------- Run backup on-demand-backup-aws-s3 for demand-backup-cloud cluster ----------------------------------------------------------------------------------- + run_backup_with_delete on-demand-backup-aws-s3 + local backup_name=on-demand-backup-aws-s3 + desc 'make backup on-demand-backup-aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- make backup on-demand-backup-aws-s3 ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml ++ mktemp + local LAST_OUT=/tmp/tmp.kTx3umlOf7 ++ mktemp + local LAST_ERR=/tmp/tmp.y3vgXckRhi + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/on-demand-backup-aws-s3.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kTx3umlOf7 perconaxtradbclusterbackup.pxc.percona.com/on-demand-backup-aws-s3 created + cat /tmp/tmp.y3vgXckRhi + rm /tmp/tmp.kTx3umlOf7 /tmp/tmp.y3vgXckRhi + return 0 + delete_backup_pod on-demand-backup-aws-s3 + local backup_name=on-demand-backup-aws-s3 + desc 'Delete on-demand-backup-aws-s3 pod during SST' + set +o xtrace ----------------------------------------------------------------------------------- Delete on-demand-backup-aws-s3 pod during SST ----------------------------------------------------------------------------------- + echo 'Waiting for on-demand-backup-aws-s3 pod to become Running' Waiting for on-demand-backup-aws-s3 pod to become Running + sleep 1 + kubectl_bin wait '--for=jsonpath={.status.phase}=Running' pod --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 --timeout=120s ++ mktemp + local LAST_OUT=/tmp/tmp.IVIXonVkr9 ++ mktemp + local LAST_ERR=/tmp/tmp.ycH1C25o8S + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait '--for=jsonpath={.status.phase}=Running' pod --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 --timeout=120s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IVIXonVkr9 pod/xb-on-demand-backup-aws-s3-b788q condition met + cat /tmp/tmp.ycH1C25o8S + rm /tmp/tmp.IVIXonVkr9 /tmp/tmp.ycH1C25o8S + return 0 ++ kubectl_bin get pods --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.skQfEItkCS +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZvnW0MM8Os ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.skQfEItkCS ++ cat /tmp/tmp.ZvnW0MM8Os ++ rm /tmp/tmp.skQfEItkCS /tmp/tmp.ZvnW0MM8Os ++ return 0 + backup_pod=xb-on-demand-backup-aws-s3-b788q + echo 'Deleting pod/xb-on-demand-backup-aws-s3-b788q during SST upload' Deleting pod/xb-on-demand-backup-aws-s3-b788q during SST upload + kubectl logs -f xb-on-demand-backup-aws-s3-b788q + IFS= + read -r line Defaulted container "xtrabackup" out of: xtrabackup, backup-init (init) + [[ + LIB_PATH=/opt/percona/backup/lib/pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ + . /opt/percona/backup/lib/pxc/backup.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ LIB_PATH=/opt/percona/backup/lib/pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ . /opt/percona/backup/lib/pxc/aws.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ export AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ export AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ export AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ caBundleDir=/etc/s3/certs =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ caBundleFile=/etc/s3/certs/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ '[' -f /etc/s3/certs/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ SST_INFO_NAME=sst_info =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ XBCLOUD_ARGS='--curl-retriable-errors=7 --parallel=2' =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ S3_BUCKET_PATH=demand-backup-cloud-2025-11-12-10:51:55-full =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ date +%F-%H-%M =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ BACKUP_PATH=demand-backup-cloud-pxc-2025-11-12-10-51-xtrabackup.stream =~ \.ibd\. ]] + IFS= + read -r line + [[ + . /opt/percona/backup/lib/pxc/vault.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ keyring_vault=/etc/mysql/vault-keyring-secret/keyring_vault.conf =~ \.ibd\. ]] + IFS= + read -r line + [[ + . /opt/percona/backup/lib/pxc/aws.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ export AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ export AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ export AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ caBundleDir=/etc/s3/certs =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ caBundleFile=/etc/s3/certs/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ '[' -f /etc/s3/certs/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + . /opt/percona/backup/lib/pxc/check-version.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ + GARBD_OPTS= =~ \.ibd\. ]] + IFS= + read -r line + [[ + check_ssl =~ \.ibd\. ]] + IFS= + read -r line + [[ + CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + SSL_DIR=/etc/mysql/ssl =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + CA=/etc/mysql/ssl/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ + SSL_INTERNAL_DIR=/etc/mysql/ssl-internal =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + CA=/etc/mysql/ssl-internal/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ + KEY=/etc/mysql/ssl/tls.key =~ \.ibd\. ]] + IFS= + read -r line + [[ + CERT=/etc/mysql/ssl/tls.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/tls.key ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/tls.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + KEY=/etc/mysql/ssl-internal/tls.key =~ \.ibd\. ]] + IFS= + read -r line + [[ + CERT=/etc/mysql/ssl-internal/tls.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/tls.key ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -f /etc/mysql/ssl-internal/tls.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + GARBD_OPTS='socket.ssl_ca=/etc/mysql/ssl-internal/ca.crt;socket.ssl_cert=/etc/mysql/ssl-internal/tls.crt;socket.ssl_key=/etc/mysql/ssl-internal/tls.key;socket.ssl_cipher=;pc.weight=0;' =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -n operator-testing ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + clean_backup_s3 =~ \.ibd\. ]] + IFS= + read -r line + [[ + s3_add_bucket_dest =~ \.ibd\. ]] + IFS= + read -r line + [[ + local time=15 =~ \.ibd\. ]] + IFS= + read -r line + [[ + local is_deleted_full=0 =~ \.ibd\. ]] + IFS= + read -r line + [[ + local is_deleted_info=0 =~ \.ibd\. ]] + IFS= + read -r line + [[ + for i in {1..5} =~ \.ibd\. ]] + IFS= + read -r line + [[ + (( i > 1 )) =~ \.ibd\. ]] + IFS= + read -r line + [[ + is_object_exist operator-testing demand-backup-cloud-2025-11-12-10:51:55-full/ =~ \.ibd\. ]] + IFS= + read -r line + [[ + local bucket=operator-testing =~ \.ibd\. ]] + IFS= + read -r line + [[ + local path=demand-backup-cloud-2025-11-12-10:51:55-full/ =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ aws s3 ls s3://operator-testing/demand-backup-cloud-2025-11-12-10:51:55-full/ --summarize --recursive =~ \.ibd\. ]] + IFS= + read -r line + [[ + res=' =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Objects: 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Size: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + echo ' =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Objects: 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Size: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + grep -q 'Total Objects: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + return 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ + is_deleted_full=1 =~ \.ibd\. ]] + IFS= + read -r line + [[ + is_object_exist operator-testing demand-backup-cloud-2025-11-12-10:51:55-full.sst_info/ =~ \.ibd\. ]] + IFS= + read -r line + [[ + local bucket=operator-testing =~ \.ibd\. ]] + IFS= + read -r line + [[ + local path=demand-backup-cloud-2025-11-12-10:51:55-full.sst_info/ =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ aws s3 ls s3://operator-testing/demand-backup-cloud-2025-11-12-10:51:55-full.sst_info/ --summarize --recursive =~ \.ibd\. ]] + IFS= + read -r line + [[ + res=' =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Objects: 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Size: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + echo ' =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Objects: 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ Total Size: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + grep -q 'Total Objects: 0' =~ \.ibd\. ]] + IFS= + read -r line + [[ + return 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ + is_deleted_info=1 =~ \.ibd\. ]] + IFS= + read -r line + [[ + [[ 1 == 1 ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ + [[ 1 == 1 ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ + log INFO 'Object deleted successfully before attempt 1. Exiting.' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:01 [INFO] Object deleted successfully before attempt 1. Exiting. =~ \.ibd\. ]] + IFS= + read -r line + [[ + break =~ \.ibd\. ]] + IFS= + read -r line + [[ + request_streaming =~ \.ibd\. ]] + IFS= + read -r line + [[ + local LOCAL_IP =~ \.ibd\. ]] + IFS= + read -r line + [[ + local NODE_NAME =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ hostname -i =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ sed -E 's/.*\b([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\b.*/\1/' =~ \.ibd\. ]] + IFS= + read -r line + [[ + LOCAL_IP=10.254.194.9 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ get_backup_source =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ /opt/percona/peer-list -on-start=/opt/percona/backup/lib/pxc/get-pxc-state.sh -service=demand-backup-cloud-pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ grep wsrep_cluster_size =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ sort =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ tail -1 =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ cut -d : -f 12 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ CLUSTER_SIZE=3 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ '[' -z 3 ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ /opt/percona/peer-list -on-start=/opt/percona/backup/lib/pxc/get-pxc-state.sh -service=demand-backup-cloud-pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ grep wsrep_ready:ON:wsrep_connected:ON:wsrep_local_state_comment:Synced:wsrep_cluster_status:Primary =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ tail -1 =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ sort -r =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ cut -d : -f 2 =~ \.ibd\. ]] + IFS= + read -r line + [[ +++ cut -d . -f 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ FIRST_NODE=demand-backup-cloud-pxc-0 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ SKIP_FIRST_POD='|' =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ (( 3 > 1 )) =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ SKIP_FIRST_POD=demand-backup-cloud-pxc-0 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ /opt/percona/peer-list -on-start=/opt/percona/backup/lib/pxc/get-pxc-state.sh -service=demand-backup-cloud-pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ grep wsrep_ready:ON:wsrep_connected:ON:wsrep_local_state_comment:Synced:wsrep_cluster_status:Primary =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ grep -v demand-backup-cloud-pxc-0 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ sort =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ tail -1 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ cut -d : -f 2 =~ \.ibd\. ]] + IFS= + read -r line + [[ ++ cut -d . -f 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ + NODE_NAME=demand-backup-cloud-pxc-2 =~ \.ibd\. ]] + IFS= + read -r line + [[ + '[' -z demand-backup-cloud-pxc-2 ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ + set +o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ + log INFO 'Garbd was started' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04 [INFO] Garbd was started =~ \.ibd\. ]] + IFS= + read -r line + [[ + garbd --address 'gcomm://demand-backup-cloud-pxc-2.demand-backup-cloud-pxc?gmcast.listen_addr=tcp://0.0.0.0:4567' --donor demand-backup-cloud-pxc-2 --group demand-backup-cloud-pxc --options 'socket.ssl_ca=/etc/mysql/ssl-internal/ca.crt;socket.ssl_cert=/etc/mysql/ssl-internal/tls.crt;socket.ssl_key=/etc/mysql/ssl-internal/tls.key;socket.ssl_cipher=;pc.weight=0;' --sst xtrabackup-v2:10.254.194.9:4444/xtrabackup_sst//1 --recv-script=/opt/percona/backup/run_backup.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ + tee /tmp/garbd.log =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.881 INFO: CRC-32C: using 64-bit x86 acceleration. =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.881 INFO: Read config: =~ \.ibd\. ]] + IFS= + read -r line + [[ daemon: 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ name: garb =~ \.ibd\. ]] + IFS= + read -r line + [[ address: gcomm://demand-backup-cloud-pxc-2.demand-backup-cloud-pxc?gmcast.listen_addr=tcp://0.0.0.0:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ group: demand-backup-cloud-pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ sst: xtrabackup-v2:10.254.194.9:4444/xtrabackup_sst//1 =~ \.ibd\. ]] + IFS= + read -r line + [[ donor: demand-backup-cloud-pxc-2 =~ \.ibd\. ]] + IFS= + read -r line + [[ options: socket.ssl_ca=/etc/mysql/ssl-internal/ca.crt;socket.ssl_cert=/etc/mysql/ssl-internal/tls.crt;socket.ssl_key=/etc/mysql/ssl-internal/tls.key;socket.ssl_cipher=;pc.weight=0;; gcs.fc_limit=9999999; gcs.fc_factor=1.0;gcs.fc_single_primary=yes; gcs.stateless=yes;; socket.ssl=YES =~ \.ibd\. ]] + IFS= + read -r line + [[ cfg: =~ \.ibd\. ]] + IFS= + read -r line + [[ log: =~ \.ibd\. ]] + IFS= + read -r line + [[ recv_script: /opt/percona/backup/run_backup.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ workdir: =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.881 INFO: not using SSL compression =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.884 INFO: protonet asio version 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: Using CRC-32C for message checksums. =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: backend: asio =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: gcomm thread scheduling priority set to other:0 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: Fail to access the file (./gvwstate.dat) error (No such file or directory). It is possible if node is booting for first time or re-booting after a graceful shutdown =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: Restoring primary-component from disk failed. Either node is booting for first time or re-booting after a graceful shutdown =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.885 INFO: GMCast version 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.886 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') listening at ssl://0.0.0.0:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.886 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') multicast: , ttl: 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.886 INFO: EVS version 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.886 INFO: gcomm: connecting to group 'demand-backup-cloud-pxc', peer 'demand-backup-cloud-pxc-2.demand-backup-cloud-pxc:' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.891 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') connection established to 4e36c601-bc91 ssl://10.254.192.9:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:04.891 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') turning message relay requesting on, nonlive peers: ssl://10.254.193.12:4567 ssl://10.254.194.8:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.083 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') connection established to 004a3e13-8d94 ssl://10.254.194.8:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.087 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') connection established to 20fca12c-8ce2 ssl://10.254.193.12:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.088 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') connection established to 20fca12c-8ce2 ssl://10.254.193.12:4567 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: EVS version upgrade 0 -> 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: declaring 004a3e13-8d94 at ssl://10.254.194.8:4567 stable =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: declaring 20fca12c-8ce2 at ssl://10.254.193.12:4567 stable =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: declaring 4e36c601-bc91 at ssl://10.254.192.9:4567 stable =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: PC protocol upgrade 0 -> 1 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.388 INFO: Node 004a3e13-8d94 state primary =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.389 INFO: Current view of cluster as seen by this node =~ \.ibd\. ]] + IFS= + read -r line + [[ view (view_id(PRIM,004a3e13-8d94,4) =~ \.ibd\. ]] + IFS= + read -r line + [[ memb { =~ \.ibd\. ]] + IFS= + read -r line + [[ 004a3e13-8d94,0 =~ \.ibd\. ]] + IFS= + read -r line + [[ 20fca12c-8ce2,0 =~ \.ibd\. ]] + IFS= + read -r line + [[ 4e36c601-bc91,0 =~ \.ibd\. ]] + IFS= + read -r line + [[ a05bfd35-9797,0 =~ \.ibd\. ]] + IFS= + read -r line + [[ } =~ \.ibd\. ]] + IFS= + read -r line + [[ joined { =~ \.ibd\. ]] + IFS= + read -r line + [[ } =~ \.ibd\. ]] + IFS= + read -r line + [[ left { =~ \.ibd\. ]] + IFS= + read -r line + [[ } =~ \.ibd\. ]] + IFS= + read -r line + [[ partitioned { =~ \.ibd\. ]] + IFS= + read -r line + [[ } =~ \.ibd\. ]] + IFS= + read -r line + [[ ) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.389 INFO: Save the discovered primary-component to disk =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.389 WARN: open file(./gvwstate.dat.tmp) failed(Permission denied) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: gcomm: connected =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: Changing maximum packet size to 64500, resulting msg size: 32636 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: Shifting CLOSED -> OPEN (TO: 0) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: Opened channel 'demand-backup-cloud-pxc' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: New COMPONENT: primary = yes, bootstrap = no, my_idx = 3, memb_num = 4 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: STATE EXCHANGE: Waiting for state UUID. =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: STATE EXCHANGE: sent state msg: a0a930ce-bfb5-11f0-be24-677ff5bdded3 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: STATE EXCHANGE: got state msg: a0a930ce-bfb5-11f0-be24-677ff5bdded3 from 0 (demand-backup-cloud-pxc-0) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: STATE EXCHANGE: got state msg: a0a930ce-bfb5-11f0-be24-677ff5bdded3 from 1 (demand-backup-cloud-pxc-1) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.887 INFO: STATE EXCHANGE: got state msg: a0a930ce-bfb5-11f0-be24-677ff5bdded3 from 2 (demand-backup-cloud-pxc-2) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.888 INFO: STATE EXCHANGE: got state msg: a0a930ce-bfb5-11f0-be24-677ff5bdded3 from 3 (garb) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.888 INFO: Quorum results: =~ \.ibd\. ]] + IFS= + read -r line + [[ version = 6, =~ \.ibd\. ]] + IFS= + read -r line + [[ component = PRIMARY, =~ \.ibd\. ]] + IFS= + read -r line + [[ conf_id = 3, =~ \.ibd\. ]] + IFS= + read -r line + [[ members = 3/4 (primary/total), =~ \.ibd\. ]] + IFS= + read -r line + [[ act_id = 38, =~ \.ibd\. ]] + IFS= + read -r line + [[ last_appl. = 20, =~ \.ibd\. ]] + IFS= + read -r line + [[ protocols = 5/11/4 (gcs/repl/appl), =~ \.ibd\. ]] + IFS= + read -r line + [[ vote policy= 0, =~ \.ibd\. ]] + IFS= + read -r line + [[ group UUID = f0a84e74-bfb4-11f0-b0c7-8f99654d11f2 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.888 INFO: Flow-control interval: [9999999, 9999999] =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.888 INFO: Shifting OPEN -> PRIMARY (TO: 39) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.888 INFO: Sending state transfer request: 'xtrabackup-v2:10.254.194.9:4444/xtrabackup_sst//1', size: 49 =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.889 INFO: Member 3.0 (garb) requested state transfer from 'demand-backup-cloud-pxc-2'. Selected 2.0 (demand-backup-cloud-pxc-2)(SYNCED) as donor. =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.889 INFO: Shifting PRIMARY -> JOINER (TO: 39) =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.889 INFO: Starting SST script =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.893 INFO: [SST script] + set -m =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.893 INFO: [SST script] + LIB_PATH=/opt/percona/backup/lib/pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.893 INFO: [SST script] + . /opt/percona/backup/lib/pxc/vault.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.893 INFO: [SST script] ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ keyring_vault=/etc/mysql/vault-keyring-secret/keyring_vault.conf =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] + . /opt/percona/backup/lib/pxc/backup.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ LIB_PATH=/opt/percona/backup/lib/pxc =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ . /opt/percona/backup/lib/pxc/aws.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ export AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ export AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ export AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ caBundleDir=/etc/s3/certs =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ caBundleFile=/etc/s3/certs/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ '[' -f /etc/s3/certs/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ SST_INFO_NAME=sst_info =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ XBCLOUD_ARGS='--curl-retriable-errors=7 --parallel=2' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] ++ S3_BUCKET_PATH=demand-backup-cloud-2025-11-12-10:51:55-full =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.894 INFO: [SST script] +++ date +%F-%H-%M =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.895 INFO: [SST script] ++ BACKUP_PATH=demand-backup-cloud-pxc-2025-11-12-10-52-xtrabackup.stream =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.895 INFO: [SST script] + . /opt/percona/backup/lib/pxc/aws.sh =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.895 INFO: [SST script] ++ set -o errexit =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.895 INFO: [SST script] ++ export AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.895 INFO: [SST script] ++ AWS_SHARED_CREDENTIALS_FILE=/tmp/aws-credfile =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ export AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ AWS_REGION=us-east-1 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ export AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ AWS_ENDPOINT_URL=https://s3.us-east-1.amazonaws.com =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ '[' -n true ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ [[ true == \f\a\l\s\e ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ caBundleDir=/etc/s3/certs =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ caBundleFile=/etc/s3/certs/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] ++ '[' -f /etc/s3/certs/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] + SOCAT_OPTS=TCP-LISTEN:4444,reuseaddr,retry=30 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] + FIRST_RECEIVED=0 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.896 INFO: [SST script] + SST_FAILED=0 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + check_ssl =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + SSL_DIR=/etc/mysql/ssl =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + CA=/etc/mysql/ssl/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + SSL_INTERNAL_DIR=/etc/mysql/ssl-internal =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + CA=/etc/mysql/ssl-internal/ca.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + KEY=/etc/mysql/ssl/tls.key =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + CERT=/etc/mysql/ssl/tls.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/tls.key ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/tls.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + KEY=/etc/mysql/ssl-internal/tls.key =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + CERT=/etc/mysql/ssl-internal/tls.crt =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/ca.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/tls.key ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -f /etc/mysql/ssl-internal/tls.crt ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + SOCAT_OPTS=openssl-listen:4444,reuseaddr,cert=/etc/mysql/ssl-internal/tls.crt,key=/etc/mysql/ssl-internal/tls.key,cafile=/etc/mysql/ssl-internal/ca.crt,verify=1,retry=30 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + trap handle_sigterm 15 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + '[' -n operator-testing ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + backup_s3 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:05.897 INFO: [SST script] + s3_add_bucket_dest =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:06.983 INFO: [SST script] + local socat_status =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:06.983 INFO: [SST script] + wait 356 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:06.983 INFO: [SST script] + xbstream -x -C /tmp --parallel=2 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:06.983 INFO: [SST script] + socat -u openssl-listen:4444,reuseaddr,cert=/etc/mysql/ssl-internal/tls.crt,key=/etc/mysql/ssl-internal/tls.key,cafile=/etc/mysql/ssl-internal/ca.crt,verify=1,retry=30 stdio =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.620 INFO: [SST script] + socat_status=0 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.620 INFO: [SST script] + log INFO 'Socat was started' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.622 INFO: [SST script] 2025-11-12 10:52:07 [INFO] Socat was started =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.622 INFO: [SST script] + FIRST_RECEIVED=1 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.622 INFO: [SST script] + [[ 0 -ne 0 ]] =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.622 INFO: [SST script] + vault_store /tmp/sst_info =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.623 INFO: [SST script] + local sst_info=/tmp/sst_info =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.623 INFO: [SST script] + '[' '!' -f /etc/mysql/vault-keyring-secret/keyring_vault.conf ']' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.623 INFO: [SST script] + echo 'vault configuration not found' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.623 INFO: [SST script] vault configuration not found =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.623 INFO: [SST script] + return 0 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.624 INFO: [SST script] + xbstream -C /tmp -c sst_info --parallel=2 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.624 INFO: [SST script] + grep -v 'error: http request failed: Couldn'\''t resolve host name' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.624 INFO: [SST script] ++ grep -c processor /proc/cpuinfo =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:07.626 INFO: [SST script] + xbcloud put --storage=s3 --parallel=4 --md5 --curl-retriable-errors=7 --parallel=2 --s3-bucket=operator-testing demand-backup-cloud-2025-11-12-10:51:55-full.sst_info =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] 251112 10:52:07 xbcloud: Successfully connected. =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] 251112 10:52:08 xbcloud: [0] successfully uploaded chunk: demand-backup-cloud-2025-11-12-10:51:55-full.sst_info/sst_info.00000000000000000001, size: 22 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] 251112 10:52:08 xbcloud: [0] successfully uploaded chunk: demand-backup-cloud-2025-11-12-10:51:55-full.sst_info/sst_info.00000000000000000000, size: 145 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] 251112 10:52:08 xbcloud: Uploading md5 =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] 251112 10:52:08 xbcloud: Upload failed: backup is incomplete. =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.364 INFO: [SST script] Backup doesn't contain last file with prefix xtrabackup_tablespaces in the cloud storage =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.365 INFO: [SST script] + (( SST_FAILED == 0 )) =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.366 INFO: [SST script] + grep -v 'error: http request failed: Couldn'\''t resolve host name' =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.366 INFO: [SST script] + socat -u openssl-listen:4444,reuseaddr,cert=/etc/mysql/ssl-internal/tls.crt,key=/etc/mysql/ssl-internal/tls.key,cafile=/etc/mysql/ssl-internal/ca.crt,verify=1,retry=30 stdio =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.366 INFO: [SST script] ++ grep -c processor /proc/cpuinfo =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.368 INFO: [SST script] + xbcloud put --storage=s3 --parallel=4 --md5 --curl-retriable-errors=7 --parallel=2 --s3-bucket=operator-testing demand-backup-cloud-2025-11-12-10:51:55-full =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:08.387 INFO: (a05bfd35-9797, 'ssl://0.0.0.0:4567') turning message relay requesting off =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:20.496 INFO: [SST script] 251112 10:52:08 xbcloud: Successfully connected. =~ \.ibd\. ]] + IFS= + read -r line + [[ '' =~ \.ibd\. ]] + IFS= + read -r line + [[ 2025-11-12 10:52:20.496 INFO: [SST script] 251112 10:52:19 xbcloud: [0] successfully uploaded chunk: demand-backup-cloud-2025-11-12-10:51:55-full/sys/sys_config.ibd.lz4.00000000000000000001, size: 36 =~ \.ibd\. ]] + kubectl delete pod --force xb-on-demand-backup-aws-s3-b788q Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. pod "xb-on-demand-backup-aws-s3-b788q" force deleted from demand-backup-cloud-27138 namespace + break + wait_backup on-demand-backup-aws-s3 + local backup=on-demand-backup-aws-s3 + local status=Succeeded + set +o xtrace waiting for pxc-backup/on-demand-backup-aws-s3 to reach Succeeded state...............................................Succeeded + check_cloud_storage_cleanup on-demand-backup-aws-s3 + local backup_name=on-demand-backup-aws-s3 + desc 'Check storage cleanup of on-demand-backup-aws-s3' + set +o xtrace ----------------------------------------------------------------------------------- Check storage cleanup of on-demand-backup-aws-s3 ----------------------------------------------------------------------------------- ++ kubectl_bin get events --field-selector involvedObject.kind=Job,involvedObject.name=xb-on-demand-backup-aws-s3 +++ mktemp ++ local LAST_OUT=/tmp/tmp.riVTK3qydV ++ grep -c 'Created pod' +++ mktemp ++ local LAST_ERR=/tmp/tmp.mPbZazoY3J ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get events --field-selector involvedObject.kind=Job,involvedObject.name=xb-on-demand-backup-aws-s3 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.riVTK3qydV ++ cat /tmp/tmp.mPbZazoY3J ++ rm /tmp/tmp.riVTK3qydV /tmp/tmp.mPbZazoY3J ++ return 0 + [[ 2 == \1 ]] ++ kubectl_bin get pods --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uANLqOvkqX +++ mktemp ++ local LAST_ERR=/tmp/tmp.YFlaOwUaIv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=percona.com/backup-job-name=xb-on-demand-backup-aws-s3 -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uANLqOvkqX ++ cat /tmp/tmp.YFlaOwUaIv ++ rm /tmp/tmp.uANLqOvkqX /tmp/tmp.YFlaOwUaIv ++ return 0 + local backup_pod=xb-on-demand-backup-aws-s3-6c7zv + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + kubectl_bin logs xb-on-demand-backup-aws-s3-6c7zv ++ mktemp + local LAST_OUT=/tmp/tmp.Ueick2xgDI + grep 'Object deleted successfully before attempt 1. Exiting.' ++ mktemp + local LAST_ERR=/tmp/tmp.JcJtThTsNo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs xb-on-demand-backup-aws-s3-6c7zv + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ueick2xgDI + cat /tmp/tmp.JcJtThTsNo Defaulted container "xtrabackup" out of: xtrabackup, backup-init (init) + rm /tmp/tmp.Ueick2xgDI /tmp/tmp.JcJtThTsNo + return 0 + echo 'Clenup was performed.' Clenup was performed. + desc 'Run recovery from s3 for demand-backup-cloud cluster' + set +o xtrace ----------------------------------------------------------------------------------- Run recovery from s3 for demand-backup-cloud cluster ----------------------------------------------------------------------------------- + run_recovery_check demand-backup-cloud on-demand-backup-aws-s3 + local cluster=demand-backup-cloud + local backup=on-demand-backup-aws-s3 ++ get_proxy_engine demand-backup-cloud ++ local cluster_name=demand-backup-cloud +++ get_proxy demand-backup-cloud +++ local target_cluster=demand-backup-cloud ++++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.VNvD5Sz0fa +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.uUTg9eFkXr ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.haproxy.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.VNvD5Sz0fa ++++ cat /tmp/tmp.uUTg9eFkXr ++++ rm /tmp/tmp.VNvD5Sz0fa /tmp/tmp.uUTg9eFkXr ++++ return 0 +++ [[ '' == \t\r\u\e ]] ++++ kubectl_bin get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qreNbfkmHH +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.EroNqE74Yw ++++ local exit_status=0 +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ set +e ++++ kubectl get pxc demand-backup-cloud -o 'jsonpath={.spec.proxysql.enabled}' ++++ exit_status=0 ++++ set -e ++++ '[' 0 '!=' 0 ']' ++++ break ++++ cat /tmp/tmp.qreNbfkmHH ++++ cat /tmp/tmp.EroNqE74Yw ++++ rm /tmp/tmp.qreNbfkmHH /tmp/tmp.EroNqE74Yw ++++ return 0 +++ [[ true == \t\r\u\e ]] +++ echo demand-backup-cloud-proxysql +++ return ++ local cluster_proxy=demand-backup-cloud-proxysql ++ echo proxysql + local proxy=proxysql + log 'run pxc-restore/on-demand-backup-aws-s3' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2025-11-12T10:53:55+0000]' run pxc-restore/on-demand-backup-aws-s3 [2025-11-12T10:53:55+0000] run pxc-restore/on-demand-backup-aws-s3 + /usr/bin/sed -e s~minio-service.#namespace~minio-service.demand-backup-cloud-27138~ + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2234/e2e-tests/demand-backup-cloud/conf/restore-on-demand-backup-aws-s3.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.z1xyAs1Lxn ++ mktemp + local LAST_ERR=/tmp/tmp.dS4NmqDQSZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.z1xyAs1Lxn perconaxtradbclusterrestore.pxc.percona.com/on-demand-backup-aws-s3 created + cat /tmp/tmp.dS4NmqDQSZ + rm /tmp/tmp.z1xyAs1Lxn /tmp/tmp.dS4NmqDQSZ + return 0 + wait_backup_restore on-demand-backup-aws-s3 + local backup_name=on-demand-backup-aws-s3 + local target_state=Succeeded + local wait_time=720 + set +o xtrace waiting for pxc-restore/on-demand-backup-aws-s3 to reach Succeeded state 2025-11-12T10:54:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:54:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:55:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:25 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:56:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:57:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:58:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T10:59:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:00:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:01:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:02:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:03:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:25 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:04:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:05:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:06:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:07:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:25 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:08:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:39 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:09:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:10:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:11:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:12 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:16 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:12:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:13:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:08 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:41 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:43 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:14:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:25 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:15:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:02 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:04 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:06 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:10 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:45 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:47 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:49 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:51 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:16:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:18 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:20 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:22 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:24 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:29 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:31 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:33 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:35 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:37 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:56 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:17:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:25 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:27 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:52 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:54 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:18:58 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:00 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:13 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:15 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:17 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:19 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:21 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:23 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:26 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:28 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:30 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:32 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:34 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:36 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:38 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:40 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:42 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:44 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:46 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:48 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:50 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:53 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:55 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:57 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:19:59 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:01 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:03 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:05 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:07 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:09 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:11 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:14 pxc-restore/on-demand-backup-aws-s3 state: Starting 2025-11-12T11:20:16 pxc-restore/on-demand-backup-aws-s3 state: Starting apiVersion: pxc.percona.com/v1 kind: PerconaXtraDBClusterRestore metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"pxc.percona.com/v1","kind":"PerconaXtraDBClusterRestore","metadata":{"annotations":{},"name":"on-demand-backup-aws-s3","namespace":"demand-backup-cloud-27138"},"spec":{"backupName":"on-demand-backup-aws-s3","containerOptions":{"args":{"xtrabackup":["--innodb-undo-directory=./undo","--defaults-file=backup-my.cnf"]}},"pxcCluster":"demand-backup-cloud"}} creationTimestamp: "2025-11-12T10:53:58Z" generation: 1 name: on-demand-backup-aws-s3 namespace: demand-backup-cloud-27138 resourceVersion: "1762944839029407018" uid: 143e4c69-e86d-408a-946a-3aa63fd08fae spec: backupName: on-demand-backup-aws-s3 containerOptions: args: xtrabackup: - --innodb-undo-directory=./undo - --defaults-file=backup-my.cnf pxcCluster: demand-backup-cloud status: state: Starting unsafeFlags: {} Restore object pxc-restore/on-demand-backup-aws-s3 is in Starting state. something went wrong with operator or kubernetes cluster