Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/logs/storage-8-0.log Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.37) and server (1.34) exceeds the supported minor version skew of +/-1 + '[' -n '' ']' + '[' 0 = 1 ']' + main + create_infra storage-14058 + local ns=storage-14058 + '[' -n pxc-operator ']' + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ awk '{print $1}' ++ grep chaos-mesh.org ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.9AfeQ4NeYC + awk '{print$1}' + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.Ppepwyswfv + local exit_status=0 + local LAST_OUT=/tmp/tmp.K8APLVSfTn ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_ERR=/tmp/tmp.a3nkT10JQg + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.K8APLVSfTn + cat /tmp/tmp.a3nkT10JQg + rm /tmp/tmp.K8APLVSfTn /tmp/tmp.a3nkT10JQg + return 0 namespace "storage-6268" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9AfeQ4NeYC namespace "pxc-operator" deleted + cat /tmp/tmp.Ppepwyswfv + rm /tmp/tmp.9AfeQ4NeYC /tmp/tmp.Ppepwyswfv + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7BANUxZGJI ++ mktemp + local LAST_ERR=/tmp/tmp.qyRbZOAOXy + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7BANUxZGJI namespace/pxc-operator created + cat /tmp/tmp.qyRbZOAOXy + rm /tmp/tmp.7BANUxZGJI /tmp/tmp.qyRbZOAOXy + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.maGO0pJ0qY +++ mktemp ++ local LAST_ERR=/tmp/tmp.hUFV6kbrSc ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.maGO0pJ0qY ++ cat /tmp/tmp.hUFV6kbrSc ++ rm /tmp/tmp.maGO0pJ0qY /tmp/tmp.hUFV6kbrSc ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.VYPDR9feV7 ++ mktemp + local LAST_ERR=/tmp/tmp.G6GLCS4bNB + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VYPDR9feV7 Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11" modified. + cat /tmp/tmp.G6GLCS4bNB + rm /tmp/tmp.VYPDR9feV7 /tmp/tmp.G6GLCS4bNB + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rKuVrsDtCg ++ mktemp + local LAST_ERR=/tmp/tmp.VMamhgysFt + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rKuVrsDtCg customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.VMamhgysFt + rm /tmp/tmp.rKuVrsDtCg /tmp/tmp.VMamhgysFt + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.0FCg5KdDgV ++ mktemp + local LAST_ERR=/tmp/tmp.CyhatNgHKL + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0FCg5KdDgV customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.CyhatNgHKL + rm /tmp/tmp.0FCg5KdDgV /tmp/tmp.CyhatNgHKL + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.Bco9XhqDQQ ++ mktemp + local LAST_ERR=/tmp/tmp.HpYvWM1VUY + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Bco9XhqDQQ customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.HpYvWM1VUY + rm /tmp/tmp.Bco9XhqDQQ /tmp/tmp.HpYvWM1VUY + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.HOixVmRF7U ++ mktemp + local LAST_ERR=/tmp/tmp.EdqeZgJeQE + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HOixVmRF7U customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.EdqeZgJeQE + rm /tmp/tmp.HOixVmRF7U /tmp/tmp.EdqeZgJeQE + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.BC1gRuN3dO ++ mktemp + local LAST_ERR=/tmp/tmp.Mjn1Qy3Phl + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.BC1gRuN3dO clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.Mjn1Qy3Phl + rm /tmp/tmp.BC1gRuN3dO /tmp/tmp.Mjn1Qy3Phl + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + local LAST_OUT=/tmp/tmp.GUMDdBMwvR + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + local LAST_ERR=/tmp/tmp.Wf0UszI0DB + local exit_status=0 + sed -e 's^failureThreshold: .*^failureThreshold: 10^' ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GUMDdBMwvR deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.Wf0UszI0DB + rm /tmp/tmp.GUMDdBMwvR /tmp/tmp.Wf0UszI0DB + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.VYEzN4LJDe ++ mktemp + local LAST_ERR=/tmp/tmp.NBrxzDqoBU + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VYEzN4LJDe pod/percona-xtradb-cluster-operator-658fcf4b98-k7dt9 condition met + cat /tmp/tmp.NBrxzDqoBU + rm /tmp/tmp.VYEzN4LJDe /tmp/tmp.NBrxzDqoBU + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.5z0rnSlXUZ +++ mktemp ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_ERR=/tmp/tmp.fnPLGW58Cx ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.5z0rnSlXUZ ++ cat /tmp/tmp.fnPLGW58Cx ++ rm /tmp/tmp.5z0rnSlXUZ /tmp/tmp.fnPLGW58Cx ++ return 0 + wait_pod percona-xtradb-cluster-operator-658fcf4b98-k7dt9 480 pxc-operator + local pod=percona-xtradb-cluster-operator-658fcf4b98-k7dt9 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-658fcf4b98-k7dt9 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-658fcf4b98-k7dt9 condition met waiting for pod/percona-xtradb-cluster-operator-658fcf4b98-k7dt9 to become Ready.Ok + sleep 3 + create_namespace storage-14058 + local namespace=storage-14058 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get clusterrolebinding ++ grep chaos-mesh + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ kubectl get clusterrole ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + '[' -n '' ']' + desc 'cleaned up old namespaces storage-14058' + set +o xtrace ++ mktemp ----------------------------------------------------------------------------------- cleaned up old namespaces storage-14058 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace storage-14058 + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' ++ mktemp + local LAST_OUT=/tmp/tmp.3eJGF9YkXN ++ mktemp + local LAST_OUT=/tmp/tmp.2v3nqD8Mls + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.qYt5LWYeNv + local exit_status=0 ++ mktemp ++ seq 0 2 + awk '{print$1}' + for i in $(seq 0 2) + set +e + kubectl get ns + local LAST_ERR=/tmp/tmp.P4fO2awGPu + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete namespace storage-14058 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace storage-14058 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3eJGF9YkXN + cat /tmp/tmp.qYt5LWYeNv + rm /tmp/tmp.3eJGF9YkXN /tmp/tmp.qYt5LWYeNv + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in $(seq 0 2) + set +e + kubectl delete namespace storage-14058 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.2v3nqD8Mls + cat /tmp/tmp.P4fO2awGPu Error from server (NotFound): namespaces "storage-14058" not found + rm /tmp/tmp.2v3nqD8Mls /tmp/tmp.P4fO2awGPu + return 1 + : + wait_for_delete namespace/storage-14058 + local res=namespace/storage-14058 + echo -n 'waiting for namespace/storage-14058 to be deleted' waiting for namespace/storage-14058 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "storage-14058" not found + desc 'create namespace storage-14058' + set +o xtrace ----------------------------------------------------------------------------------- create namespace storage-14058 ----------------------------------------------------------------------------------- + kubectl_bin create namespace storage-14058 ++ mktemp + local LAST_OUT=/tmp/tmp.uV7dkF1SvC ++ mktemp + local LAST_ERR=/tmp/tmp.FnurvutI3j + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl create namespace storage-14058 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uV7dkF1SvC namespace/storage-14058 created + cat /tmp/tmp.FnurvutI3j + rm /tmp/tmp.uV7dkF1SvC /tmp/tmp.FnurvutI3j + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.4iyPSLlAw3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.XeBgNKeWyC ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4iyPSLlAw3 ++ cat /tmp/tmp.XeBgNKeWyC ++ rm /tmp/tmp.4iyPSLlAw3 /tmp/tmp.XeBgNKeWyC ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11 --namespace=storage-14058 ++ mktemp + local LAST_OUT=/tmp/tmp.DML27Nxgds ++ mktemp + local LAST_ERR=/tmp/tmp.BMfl5QYZEN + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11 --namespace=storage-14058 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.DML27Nxgds Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2546-caf25420-2-cluster11" modified. + cat /tmp/tmp.BMfl5QYZEN + rm /tmp/tmp.DML27Nxgds /tmp/tmp.BMfl5QYZEN + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.gJGvIHMMlp ++ mktemp + local LAST_ERR=/tmp/tmp.pkHzwoojuS + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gJGvIHMMlp secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.pkHzwoojuS + rm /tmp/tmp.gJGvIHMMlp /tmp/tmp.pkHzwoojuS + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.VJAh0LfOnV + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.storage-14058~ + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + local LAST_ERR=/tmp/tmp.vTCGGFtuke + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VJAh0LfOnV deployment.apps/pxc-client created + cat /tmp/tmp.vTCGGFtuke + rm /tmp/tmp.VJAh0LfOnV /tmp/tmp.vTCGGFtuke + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath-helper.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MqBqvswNvK ++ mktemp + local LAST_ERR=/tmp/tmp.oLo3flATVq + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath-helper.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.MqBqvswNvK secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created daemonset.apps/hostpath-helper created + cat /tmp/tmp.oLo3flATVq + rm /tmp/tmp.MqBqvswNvK /tmp/tmp.oLo3flATVq + return 0 + desc 'check emptydir' + set +o xtrace ----------------------------------------------------------------------------------- check emptydir ----------------------------------------------------------------------------------- + check_cr_config emptydir + local cluster=emptydir + desc 'apply cr config' + set +o xtrace ----------------------------------------------------------------------------------- apply cr config ----------------------------------------------------------------------------------- + spinup_pxc emptydir /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + local cluster=emptydir + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.cC8ShCPHz8 ++ mktemp + local LAST_ERR=/tmp/tmp.u5naTI784u + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cC8ShCPHz8 secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.u5naTI784u + rm /tmp/tmp.cC8ShCPHz8 /tmp/tmp.u5naTI784u + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.storage-14058~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ mktemp + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + local LAST_OUT=/tmp/tmp.VAvWcaS8hc ++ mktemp + local LAST_ERR=/tmp/tmp.XVDqhjjADf + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VAvWcaS8hc deployment.apps/pxc-client unchanged + cat /tmp/tmp.XVDqhjjADf + rm /tmp/tmp.VAvWcaS8hc /tmp/tmp.XVDqhjjADf + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + local pvc_name= ++ mktemp + local LAST_OUT=/tmp/tmp.iflr9Wzxem + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.storage-14058~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + local LAST_ERR=/tmp/tmp.fkx1ZYKvTI + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iflr9Wzxem perconaxtradbcluster.pxc.percona.com/emptydir created + cat /tmp/tmp.fkx1ZYKvTI + rm /tmp/tmp.iflr9Wzxem /tmp/tmp.fkx1ZYKvTI + return 0 + desc 'waiting pods to be running for emptydir' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for emptydir ----------------------------------------------------------------------------------- ++ get_proxy emptydir ++ local target_cluster=emptydir +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vjr32i0iA9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QgPEZPkXIh +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vjr32i0iA9 +++ cat /tmp/tmp.QgPEZPkXIh +++ rm /tmp/tmp.vjr32i0iA9 /tmp/tmp.QgPEZPkXIh +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IuDRRIiDlC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.D7iEylFBNV +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.IuDRRIiDlC +++ cat /tmp/tmp.D7iEylFBNV +++ rm /tmp/tmp.IuDRRIiDlC /tmp/tmp.D7iEylFBNV +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo emptydir-proxysql ++ return + local proxy=emptydir-proxysql ++ get_proxy_size emptydir ++ local cluster=emptydir ++ local size=0 +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UCFpyWlCZf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GEfDIrcaLT +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.UCFpyWlCZf +++ cat /tmp/tmp.GEfDIrcaLT +++ rm /tmp/tmp.UCFpyWlCZf /tmp/tmp.GEfDIrcaLT +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HyaTdmtvzZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.jHxBm71lOn +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HyaTdmtvzZ +++ cat /tmp/tmp.jHxBm71lOn +++ rm /tmp/tmp.HyaTdmtvzZ /tmp/tmp.jHxBm71lOn +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CwxP3r2fvV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gHCjMKf1eM +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CwxP3r2fvV +++ cat /tmp/tmp.gHCjMKf1eM +++ rm /tmp/tmp.CwxP3r2fvV /tmp/tmp.gHCjMKf1eM +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running emptydir-pxc 3 + local name=emptydir-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z emptydir-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for emptydir-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for emptydir-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod emptydir-pxc-0 480 + local pod=emptydir-pxc-0 + local max_retry=480 + local ns= ++ echo emptydir-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/emptydir-pxc-0 condition met waiting for pod/emptydir-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod emptydir-pxc-1 480 + local pod=emptydir-pxc-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo emptydir-pxc-1 + local container=pxc + set +o xtrace pod/emptydir-pxc-1 condition met waiting for pod/emptydir-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod emptydir-pxc-2 480 + local pod=emptydir-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo emptydir-pxc-2 + local container=pxc + set +o xtrace pod/emptydir-pxc-2 condition met waiting for pod/emptydir-pxc-2 to become Ready.Ok + wait_for_running emptydir-proxysql 2 + local name=emptydir-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z emptydir-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for emptydir-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for emptydir-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod emptydir-proxysql-0 480 + local pod=emptydir-proxysql-0 + local max_retry=480 + local ns= ++ echo emptydir-proxysql-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/emptydir-proxysql-0 condition met waiting for pod/emptydir-proxysql-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod emptydir-proxysql-1 480 + local pod=emptydir-proxysql-1 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo emptydir-proxysql-1 + local container=proxysql + set +o xtrace pod/emptydir-proxysql-1 condition met waiting for pod/emptydir-proxysql-1 to become Ready.Ok + wait_cluster_consistency emptydir 3 2 + local cluster_name=emptydir + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/emptydir to be ready' waiting for pxc/emptydir to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc emptydir -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.27ht5FBX1B +++ mktemp ++ local LAST_ERR=/tmp/tmp.TXqarjID6O ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc emptydir -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.27ht5FBX1B ++ cat /tmp/tmp.TXqarjID6O ++ rm /tmp/tmp.27ht5FBX1B /tmp/tmp.TXqarjID6O ++ return 0 + state=ready ++ kubectl_bin get pxc emptydir -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vU4qW7aYWq +++ mktemp ++ local LAST_ERR=/tmp/tmp.YXTcRHBuUN ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc emptydir -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.vU4qW7aYWq ++ cat /tmp/tmp.YXTcRHBuUN ++ rm /tmp/tmp.vU4qW7aYWq /tmp/tmp.YXTcRHBuUN ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine emptydir +++ local cluster_name=emptydir ++++ get_proxy emptydir ++++ local target_cluster=emptydir +++++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.8aYSGPzkkj ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.roOl7hL9Gw +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.8aYSGPzkkj +++++ cat /tmp/tmp.roOl7hL9Gw +++++ rm /tmp/tmp.8aYSGPzkkj /tmp/tmp.roOl7hL9Gw +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.xmlpmcYWCO ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.51KdvecgrC +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.xmlpmcYWCO +++++ cat /tmp/tmp.51KdvecgrC +++++ rm /tmp/tmp.xmlpmcYWCO /tmp/tmp.51KdvecgrC +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo emptydir-proxysql ++++ return +++ local cluster_proxy=emptydir-proxysql +++ echo proxysql ++ kubectl_bin get pxc emptydir -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.PJk8zBQNOU +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZLanzqmm6b ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc emptydir -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.PJk8zBQNOU ++ cat /tmp/tmp.ZLanzqmm6b ++ rm /tmp/tmp.PJk8zBQNOU /tmp/tmp.ZLanzqmm6b ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc emptydir -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.z7ChVFhDTq +++ mktemp ++ local LAST_ERR=/tmp/tmp.VMH7UGGNEF ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.z7ChVFhDTq ++ cat /tmp/tmp.VMH7UGGNEF ++ rm /tmp/tmp.z7ChVFhDTq /tmp/tmp.VMH7UGGNEF ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h emptydir-pxc-0.emptydir-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h emptydir-pxc-0.emptydir-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AIkNs2Nu2F +++ mktemp ++ local LAST_ERR=/tmp/tmp.SoO3KKqKfB ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AIkNs2Nu2F ++ cat /tmp/tmp.SoO3KKqKfB ++ rm /tmp/tmp.AIkNs2Nu2F /tmp/tmp.SoO3KKqKfB ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-xgms2 + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data emptydir 100500 no-proxy -P3306 + local cluster=emptydir + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password emptydir root ++ local cluster=emptydir ++ local user=root +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CYKH73azRd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YwzU6zHH3F +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CYKH73azRd +++ cat /tmp/tmp.YwzU6zHH3F +++ rm /tmp/tmp.CYKH73azRd /tmp/tmp.YwzU6zHH3F +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.1RoFhFGSiL +++ mktemp ++ local LAST_ERR=/tmp/tmp.9PGIF6tVXp ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1RoFhFGSiL ++ cat /tmp/tmp.9PGIF6tVXp ++ rm /tmp/tmp.1RoFhFGSiL /tmp/tmp.9PGIF6tVXp ++ return 0 + root_pass=root_password ++ get_proxy emptydir ++ local target_cluster=emptydir +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6ORM2lF4cf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BEuq6cEyDV +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.6ORM2lF4cf +++ cat /tmp/tmp.BEuq6cEyDV +++ rm /tmp/tmp.6ORM2lF4cf /tmp/tmp.BEuq6cEyDV +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nDd4FfJc8T ++++ mktemp +++ local LAST_ERR=/tmp/tmp.257ScZ5Zbi +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.nDd4FfJc8T +++ cat /tmp/tmp.257ScZ5Zbi +++ rm /tmp/tmp.nDd4FfJc8T /tmp/tmp.257ScZ5Zbi +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo emptydir-proxysql ++ return + proxy=emptydir-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h emptydir-pxc-0.emptydir-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QEAbvoB10Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.4LkUrM7k3T ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.QEAbvoB10Z ++ cat /tmp/tmp.4LkUrM7k3T ++ rm /tmp/tmp.QEAbvoB10Z /tmp/tmp.4LkUrM7k3T ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data emptydir select-1 no-proxy -P3306 + local cluster=emptydir + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password emptydir root ++ local cluster=emptydir ++ local user=root +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uYsaO4cR4W ++++ mktemp +++ local LAST_ERR=/tmp/tmp.x8psmQfxtW +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.uYsaO4cR4W +++ cat /tmp/tmp.x8psmQfxtW +++ rm /tmp/tmp.uYsaO4cR4W /tmp/tmp.x8psmQfxtW +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.YZnFMST5lx +++ mktemp ++ local LAST_ERR=/tmp/tmp.oVPjKQycVT ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YZnFMST5lx ++ cat /tmp/tmp.oVPjKQycVT ++ rm /tmp/tmp.YZnFMST5lx /tmp/tmp.oVPjKQycVT ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size emptydir ++ local cluster=emptydir ++ local size=0 +++ kubectl_bin get pxc emptydir -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0SFtTdEVIQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.K3JATDKftV +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc emptydir -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.0SFtTdEVIQ +++ cat /tmp/tmp.K3JATDKftV +++ rm /tmp/tmp.0SFtTdEVIQ /tmp/tmp.K3JATDKftV +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-0.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.q7nZJPo1mF +++ mktemp ++ local LAST_ERR=/tmp/tmp.QQzc5ZHOaR ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.q7nZJPo1mF ++ cat /tmp/tmp.QQzc5ZHOaR ++ rm /tmp/tmp.q7nZJPo1mF /tmp/tmp.QQzc5ZHOaR ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-1.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-1.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-1.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-1.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.t8Xx0ILEyk +++ mktemp ++ local LAST_ERR=/tmp/tmp.nIO35pHor0 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.t8Xx0ILEyk ++ cat /tmp/tmp.nIO35pHor0 ++ rm /tmp/tmp.t8Xx0ILEyk /tmp/tmp.nIO35pHor0 ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-2.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-2.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h emptydir-pxc-2.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h emptydir-pxc-2.emptydir-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OnC7qdVQxj +++ mktemp ++ local LAST_ERR=/tmp/tmp.ztG0oeIbU5 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OnC7qdVQxj ++ cat /tmp/tmp.ztG0oeIbU5 ++ rm /tmp/tmp.OnC7qdVQxj /tmp/tmp.ztG0oeIbU5 ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + is_keyring_plugin_in_use emptydir + local cluster=emptydir + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + kubectl exec emptydir-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + grep -E -o 'early-plugin-load=keyring_\w+.so' + return 1 + desc 'check if statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/emptydir-pxc + local resource=statefulset/emptydir-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc.yml + local new_result=/tmp/tmp.E6q2wqkgYo/statefulset_emptydir-pxc.yml + desc 'compare statefulset/emptydir-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/emptydir-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/emptydir-pxc + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("storage-14058", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.0nm6xIUAIZ ++ mktemp + local LAST_ERR=/tmp/tmp.Y5UsPRLXXD + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/emptydir-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0nm6xIUAIZ + cat /tmp/tmp.Y5UsPRLXXD + rm /tmp/tmp.0nm6xIUAIZ /tmp/tmp.Y5UsPRLXXD + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-pxc-k127.yml /tmp/tmp.E6q2wqkgYo/statefulset_emptydir-pxc.yml + log 'compare_kubectl: statefulset/emptydir-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:32:47+0000]' compare_kubectl: statefulset/emptydir-pxc OK [2026-10-05T14:32:47+0000] compare_kubectl: statefulset/emptydir-pxc OK + compare_kubectl statefulset/emptydir-proxysql + local resource=statefulset/emptydir-proxysql + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql.yml + local new_result=/tmp/tmp.E6q2wqkgYo/statefulset_emptydir-proxysql.yml + desc 'compare statefulset/emptydir-proxysql-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/emptydir-proxysql- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.34 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k129.yml ']' + version_gt 1.27 ++ echo '1.34 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127-aks.yml ']' + kubectl_bin get -o yaml statefulset/emptydir-proxysql + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("storage-14058", "namespace") | ++ mktemp (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + local LAST_OUT=/tmp/tmp.ZX9LYK1X87 ++ mktemp + local LAST_ERR=/tmp/tmp.ap3nRC8k42 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/emptydir-proxysql + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ZX9LYK1X87 + cat /tmp/tmp.ap3nRC8k42 + rm /tmp/tmp.ZX9LYK1X87 /tmp/tmp.ap3nRC8k42 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_emptydir-proxysql-k127.yml /tmp/tmp.E6q2wqkgYo/statefulset_emptydir-proxysql.yml + log 'compare_kubectl: statefulset/emptydir-proxysql OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:32:49+0000]' compare_kubectl: statefulset/emptydir-proxysql OK [2026-10-05T14:32:49+0000] compare_kubectl: statefulset/emptydir-proxysql OK + desc 'delete cluster' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml ++ mktemp + local LAST_OUT=/tmp/tmp.NQnK4Tb7qo ++ mktemp + local LAST_ERR=/tmp/tmp.PH6zRFmC5c + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/emptydir.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NQnK4Tb7qo perconaxtradbcluster.pxc.percona.com "emptydir" deleted from storage-14058 namespace + cat /tmp/tmp.PH6zRFmC5c + rm /tmp/tmp.NQnK4Tb7qo /tmp/tmp.PH6zRFmC5c + return 0 + desc 'check hostpath' + set +o xtrace ----------------------------------------------------------------------------------- check hostpath ----------------------------------------------------------------------------------- + check_cr_config hostpath + local cluster=hostpath + desc 'apply cr config' + set +o xtrace ----------------------------------------------------------------------------------- apply cr config ----------------------------------------------------------------------------------- + spinup_pxc hostpath /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + local cluster=hostpath + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ijJGYd5W0M ++ mktemp + local LAST_ERR=/tmp/tmp.rhOMVllJua + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.ijJGYd5W0M secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.rhOMVllJua + rm /tmp/tmp.ijJGYd5W0M /tmp/tmp.rhOMVllJua + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.storage-14058~ + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_OUT=/tmp/tmp.U3fGN0mgcy ++ mktemp + local LAST_ERR=/tmp/tmp.u1AXGRGCHc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.U3fGN0mgcy deployment.apps/pxc-client unchanged + cat /tmp/tmp.u1AXGRGCHc + rm /tmp/tmp.U3fGN0mgcy /tmp/tmp.u1AXGRGCHc + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2546-caf25420#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.storage-14058~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + local LAST_OUT=/tmp/tmp.PgOOYE7TMh ++ mktemp + local LAST_ERR=/tmp/tmp.D5txcAZGbT + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PgOOYE7TMh perconaxtradbcluster.pxc.percona.com/hostpath created + cat /tmp/tmp.D5txcAZGbT + rm /tmp/tmp.PgOOYE7TMh /tmp/tmp.D5txcAZGbT + return 0 + desc 'waiting pods to be running for hostpath' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for hostpath ----------------------------------------------------------------------------------- ++ get_proxy hostpath ++ local target_cluster=hostpath +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.u1QL4LV4ES ++++ mktemp +++ local LAST_ERR=/tmp/tmp.s1ShPhIi0F +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.u1QL4LV4ES +++ cat /tmp/tmp.s1ShPhIi0F +++ rm /tmp/tmp.u1QL4LV4ES /tmp/tmp.s1ShPhIi0F +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GZv5L6LX8G ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FQszNDqgJe +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.GZv5L6LX8G +++ cat /tmp/tmp.FQszNDqgJe +++ rm /tmp/tmp.GZv5L6LX8G /tmp/tmp.FQszNDqgJe +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo hostpath-proxysql ++ return + local proxy=hostpath-proxysql ++ get_proxy_size hostpath ++ local cluster=hostpath ++ local size=0 +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vt06FiW4vq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bZjg4C0NsS +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vt06FiW4vq +++ cat /tmp/tmp.bZjg4C0NsS +++ rm /tmp/tmp.vt06FiW4vq /tmp/tmp.bZjg4C0NsS +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HpaQGc4DOj ++++ mktemp +++ local LAST_ERR=/tmp/tmp.k6v2D9C61U +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.HpaQGc4DOj +++ cat /tmp/tmp.k6v2D9C61U +++ rm /tmp/tmp.HpaQGc4DOj /tmp/tmp.k6v2D9C61U +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.idNuOmz7vu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JXL5nrrTys +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.idNuOmz7vu +++ cat /tmp/tmp.JXL5nrrTys +++ rm /tmp/tmp.idNuOmz7vu /tmp/tmp.JXL5nrrTys +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running hostpath-pxc 3 + local name=hostpath-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z hostpath-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for hostpath-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for hostpath-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in $(seq 0 $last_pod) + wait_pod hostpath-pxc-0 480 + local pod=hostpath-pxc-0 + local max_retry=480 + local ns= ++ echo hostpath-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/hostpath-pxc-0 condition met waiting for pod/hostpath-pxc-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod hostpath-pxc-1 480 + local pod=hostpath-pxc-1 + local max_retry=480 + local ns= ++ echo hostpath-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/hostpath-pxc-1 condition met waiting for pod/hostpath-pxc-1 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod hostpath-pxc-2 480 + local pod=hostpath-pxc-2 + local max_retry=480 + local ns= ++ echo hostpath-pxc-2 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/hostpath-pxc-2 condition met waiting for pod/hostpath-pxc-2 to become Ready.Ok + wait_for_running hostpath-proxysql 2 + local name=hostpath-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z hostpath-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for hostpath-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for hostpath-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in $(seq 0 $last_pod) + wait_pod hostpath-proxysql-0 480 + local pod=hostpath-proxysql-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo hostpath-proxysql-0 + local container=proxysql + set +o xtrace pod/hostpath-proxysql-0 condition met waiting for pod/hostpath-proxysql-0 to become Ready.Ok + for i in $(seq 0 $last_pod) + wait_pod hostpath-proxysql-1 480 + local pod=hostpath-proxysql-1 + local max_retry=480 + local ns= ++ echo hostpath-proxysql-1 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/hostpath-proxysql-1 condition met waiting for pod/hostpath-proxysql-1 to become Ready.Ok + wait_cluster_consistency hostpath 3 2 + local cluster_name=hostpath + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/hostpath to be ready' waiting for pxc/hostpath to be ready++ seq 1 300 + for _ in $(seq 1 "${retries}") ++ kubectl_bin get pxc hostpath -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.n9EcBPIP13 +++ mktemp ++ local LAST_ERR=/tmp/tmp.0UrY7hTP2p ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc hostpath -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.n9EcBPIP13 ++ cat /tmp/tmp.0UrY7hTP2p ++ rm /tmp/tmp.n9EcBPIP13 /tmp/tmp.0UrY7hTP2p ++ return 0 + state=ready ++ kubectl_bin get pxc hostpath -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7MToYWISpE +++ mktemp ++ local LAST_ERR=/tmp/tmp.hAWfc05su6 ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc hostpath -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.7MToYWISpE ++ cat /tmp/tmp.hAWfc05su6 ++ rm /tmp/tmp.7MToYWISpE /tmp/tmp.hAWfc05su6 ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine hostpath +++ local cluster_name=hostpath ++++ get_proxy hostpath ++++ local target_cluster=hostpath +++++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.GcR22TENaX ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.zwRMAyEy9Z +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.GcR22TENaX +++++ cat /tmp/tmp.zwRMAyEy9Z +++++ rm /tmp/tmp.GcR22TENaX /tmp/tmp.zwRMAyEy9Z +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.3zN3ktzqQf ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.D3HOTzuJMA +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in $(seq 0 2) +++++ set +e +++++ kubectl get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.3zN3ktzqQf +++++ cat /tmp/tmp.D3HOTzuJMA +++++ rm /tmp/tmp.3zN3ktzqQf /tmp/tmp.D3HOTzuJMA +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo hostpath-proxysql ++++ return +++ local cluster_proxy=hostpath-proxysql +++ echo proxysql ++ kubectl_bin get pxc hostpath -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.e1OZlpdWb6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.I6VPPISPpC ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pxc hostpath -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.e1OZlpdWb6 ++ cat /tmp/tmp.I6VPPISPpC ++ rm /tmp/tmp.e1OZlpdWb6 /tmp/tmp.I6VPPISPpC ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc hostpath -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pGTrbwa3Jt ++ base64 --decode +++ mktemp ++ local LAST_ERR=/tmp/tmp.7Hvceai1bd ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pGTrbwa3Jt ++ cat /tmp/tmp.7Hvceai1bd ++ rm /tmp/tmp.pGTrbwa3Jt /tmp/tmp.7Hvceai1bd ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h hostpath-pxc-0.hostpath-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h hostpath-pxc-0.hostpath-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F46gz25GrN +++ mktemp ++ local LAST_ERR=/tmp/tmp.JCsLDKTU7P ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F46gz25GrN ++ cat /tmp/tmp.JCsLDKTU7P ++ rm /tmp/tmp.F46gz25GrN /tmp/tmp.JCsLDKTU7P ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-xgms2 + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data hostpath 100500 no-proxy -P3306 + local cluster=hostpath + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password hostpath root ++ local cluster=hostpath ++ local user=root +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rft4qZ5Ry5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6Qo2IOkIbH +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.rft4qZ5Ry5 +++ cat /tmp/tmp.6Qo2IOkIbH +++ rm /tmp/tmp.rft4qZ5Ry5 /tmp/tmp.6Qo2IOkIbH +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.leGjxr4yDk +++ mktemp ++ local LAST_ERR=/tmp/tmp.k5SoEvvrmu ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.leGjxr4yDk ++ cat /tmp/tmp.k5SoEvvrmu ++ rm /tmp/tmp.leGjxr4yDk /tmp/tmp.k5SoEvvrmu ++ return 0 + root_pass=root_password ++ get_proxy hostpath ++ local target_cluster=hostpath +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CeI2yQ4CBZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZSeO9QIzUM +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CeI2yQ4CBZ +++ cat /tmp/tmp.ZSeO9QIzUM +++ rm /tmp/tmp.CeI2yQ4CBZ /tmp/tmp.ZSeO9QIzUM +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qYox7Gdnls ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kD2cEopUSh +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qYox7Gdnls +++ cat /tmp/tmp.kD2cEopUSh +++ rm /tmp/tmp.qYox7Gdnls /tmp/tmp.kD2cEopUSh +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo hostpath-proxysql ++ return + proxy=hostpath-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h hostpath-pxc-0.hostpath-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sTun4GcRr7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MDKFMTjTuW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.sTun4GcRr7 ++ cat /tmp/tmp.MDKFMTjTuW ++ rm /tmp/tmp.sTun4GcRr7 /tmp/tmp.MDKFMTjTuW ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6fc75f888d-xgms2 + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data hostpath select-1 no-proxy -P3306 + local cluster=hostpath + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password hostpath root ++ local cluster=hostpath ++ local user=root +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.9F4OawiHeR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vbxaZbV10P +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.9F4OawiHeR +++ cat /tmp/tmp.vbxaZbV10P +++ rm /tmp/tmp.9F4OawiHeR /tmp/tmp.vbxaZbV10P +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.ySrYSM9Ho0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.dfA3Yat2yy ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ySrYSM9Ho0 ++ cat /tmp/tmp.dfA3Yat2yy ++ rm /tmp/tmp.ySrYSM9Ho0 /tmp/tmp.dfA3Yat2yy ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size hostpath ++ local cluster=hostpath ++ local size=0 +++ kubectl_bin get pxc hostpath -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.npEZf4S2eI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.a058N1Q43A +++ local exit_status=0 ++++ seq 0 2 +++ for i in $(seq 0 2) +++ set +e +++ kubectl get pxc hostpath -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.npEZf4S2eI +++ cat /tmp/tmp.a058N1Q43A +++ rm /tmp/tmp.npEZf4S2eI /tmp/tmp.a058N1Q43A +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-0.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8jMVpDYPGW +++ mktemp ++ local LAST_ERR=/tmp/tmp.33NpuV7sNP ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.8jMVpDYPGW ++ cat /tmp/tmp.33NpuV7sNP ++ rm /tmp/tmp.8jMVpDYPGW /tmp/tmp.33NpuV7sNP ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-1.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-1.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-1.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-1.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.H5Rt6m0gUv +++ mktemp ++ local LAST_ERR=/tmp/tmp.eXTuxs3UoW ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.H5Rt6m0gUv ++ cat /tmp/tmp.eXTuxs3UoW ++ rm /tmp/tmp.H5Rt6m0gUv /tmp/tmp.eXTuxs3UoW ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + for i in $(seq 0 $((size - 1))) + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-2.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-2.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h hostpath-pxc-2.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h hostpath-pxc-2.hostpath-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p8e5qZ5Nms +++ mktemp ++ local LAST_ERR=/tmp/tmp.BMczdtknNE ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.p8e5qZ5Nms ++ cat /tmp/tmp.BMczdtknNE ++ rm /tmp/tmp.p8e5qZ5Nms /tmp/tmp.BMczdtknNE ++ return 0 + client_pod=pxc-client-6fc75f888d-xgms2 + wait_pod pxc-client-6fc75f888d-xgms2 + local pod=pxc-client-6fc75f888d-xgms2 + local max_retry=480 + local ns= ++ echo pxc-client-6fc75f888d-xgms2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6fc75f888d-xgms2 condition met waiting for pod/pxc-client-6fc75f888d-xgms2 to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.E6q2wqkgYo/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.E6q2wqkgYo/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/select-1.sql /tmp/tmp.E6q2wqkgYo/select-1.sql + is_keyring_plugin_in_use hostpath + local cluster=hostpath + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec hostpath-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + desc 'check if statefulset created with expected config' + set +o xtrace ----------------------------------------------------------------------------------- check if statefulset created with expected config ----------------------------------------------------------------------------------- + compare_kubectl statefulset/hostpath-pxc + local resource=statefulset/hostpath-pxc + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc.yml + local new_result=/tmp/tmp.E6q2wqkgYo/statefulset_hostpath-pxc.yml + desc 'compare statefulset/hostpath-pxc-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/hostpath-pxc- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k133.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | + kubectl_bin get -o yaml statefulset/hostpath-pxc (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("storage-14058", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - ++ mktemp + local LAST_OUT=/tmp/tmp.IgVgw4T1Cn ++ mktemp + local LAST_ERR=/tmp/tmp.7cQpfsxsrc + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/hostpath-pxc + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.IgVgw4T1Cn + cat /tmp/tmp.7cQpfsxsrc + rm /tmp/tmp.IgVgw4T1Cn /tmp/tmp.7cQpfsxsrc + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-pxc-k127.yml /tmp/tmp.E6q2wqkgYo/statefulset_hostpath-pxc.yml + log 'compare_kubectl: statefulset/hostpath-pxc OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:39:30+0000]' compare_kubectl: statefulset/hostpath-pxc OK [2026-10-05T14:39:30+0000] compare_kubectl: statefulset/hostpath-pxc OK + compare_kubectl statefulset/hostpath-proxysql + local resource=statefulset/hostpath-proxysql + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql.yml + local new_result=/tmp/tmp.E6q2wqkgYo/statefulset_hostpath-proxysql.yml + desc 'compare statefulset/hostpath-proxysql-' + set +o xtrace ----------------------------------------------------------------------------------- compare statefulset/hostpath-proxysql- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.34 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.34 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.34 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127.yml ']' + expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127.yml + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127-oc.yml ']' + version_gt 1.29 ++ echo '1.34 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("storage-14058", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml statefulset/hostpath-proxysql ++ mktemp + local LAST_OUT=/tmp/tmp.CbXhJt8c79 ++ mktemp + local LAST_ERR=/tmp/tmp.B9qaZbAUsy + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl get -o yaml statefulset/hostpath-proxysql + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CbXhJt8c79 + cat /tmp/tmp.B9qaZbAUsy + rm /tmp/tmp.CbXhJt8c79 /tmp/tmp.B9qaZbAUsy + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/compare/statefulset_hostpath-proxysql-k127.yml /tmp/tmp.E6q2wqkgYo/statefulset_hostpath-proxysql.yml + log 'compare_kubectl: statefulset/hostpath-proxysql OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-10-05T14:39:34+0000]' compare_kubectl: statefulset/hostpath-proxysql OK [2026-10-05T14:39:34+0000] compare_kubectl: statefulset/hostpath-proxysql OK + desc 'delete cluster' + set +o xtrace ----------------------------------------------------------------------------------- delete cluster ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml ++ mktemp + local LAST_OUT=/tmp/tmp.GY1UIjo72R ++ mktemp + local LAST_ERR=/tmp/tmp.TwXOzhCoPt + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2546/e2e-tests/storage/conf/hostpath.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GY1UIjo72R perconaxtradbcluster.pxc.percona.com "hostpath" deleted from storage-14058 namespace + cat /tmp/tmp.TwXOzhCoPt + rm /tmp/tmp.GY1UIjo72R /tmp/tmp.TwXOzhCoPt + return 0 + destroy storage-14058 + local namespace=storage-14058 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v level=info + sort -u + tee /tmp/tmp.E6q2wqkgYo/operator.log + grep -v 'get backup status: Job.batch' + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fNsJ8ZVBA8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.wRAH4yDmQC ++ local exit_status=0 +++ seq 0 2 ++ for i in $(seq 0 2) ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fNsJ8ZVBA8 ++ cat /tmp/tmp.wRAH4yDmQC ++ rm /tmp/tmp.fNsJ8ZVBA8 /tmp/tmp.wRAH4yDmQC ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-k7dt9 ++ mktemp + local LAST_OUT=/tmp/tmp.oegCnCn1KF ++ mktemp + local LAST_ERR=/tmp/tmp.nOjOiCaNga + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-658fcf4b98-k7dt9 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oegCnCn1KF + cat /tmp/tmp.nOjOiCaNga + rm /tmp/tmp.oegCnCn1KF /tmp/tmp.nOjOiCaNga + return 0 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:861 2026-10-05T14:26:41.493Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-10-05T14:26:41.493Z INFO setup Manager starting up {"gitCommit": "caf25420ee3d0d26f55c22c9a7661405b27337d1", "gitBranch": "PR-2546-caf25420", "buildTime": "2026-10-05T11:36:48Z", "goVersion": "go1.26.8", "os": "linux", "arch": "amd64"} 2026-10-05T14:26:41.493Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.34.9-gke.1655001"} 2026-10-05T14:26:41.496Z INFO setup Registering Components. 2026-10-05T14:26:42.527Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-10-05T14:26:42.527Z INFO controller-runtime.metrics Starting metrics server 2026-10-05T14:26:42.527Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-10-05T14:26:42.527Z INFO controller-runtime.webhook Starting webhook server 2026-10-05T14:26:42.527Z INFO setup Starting the Cmd. 2026-10-05T14:26:42.527Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-10-05T14:26:42.528Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-10-05T14:26:42.528Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-10-05T14:26:42.528Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-10-05T14:26:42.628Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:26:42.658Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-10-05T14:26:42.658Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-10-05T14:26:42.659Z DEBUG events percona-xtradb-cluster-operator-658fcf4b98-k7dt9_32a307ae-bad8-4c11-8bab-1ff034a61ab7 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"1385b9ed-8ceb-4d24-81bb-6b1ac1141f0d","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1791210402652031009"}, "reason": "LeaderElection"} 2026-10-05T14:26:42.659Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-10-05T14:26:42.659Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-10-05T14:26:42.659Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-10-05T14:26:42.760Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-10-05T14:26:42.760Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-10-05T14:26:42.859Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-10-05T14:26:42.859Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-10-05T14:26:42.859Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-10-05T14:26:42.859Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-10-05T14:27:34.838Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "version": "1.21.0"} 2026-10-05T14:27:35.276Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:27:35.331Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:27:35.402Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:27:35.464Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:27:35.547Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:27:35.748Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a26c0eca-9cb1-44dc-8c04-f87ab8843284", "object": "emptydir-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:27:36.645Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "05611b7a-1358-4524-970b-aec41c46a17b", "object": "emptydir-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:27:36.687Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "05611b7a-1358-4524-970b-aec41c46a17b", "object": "emptydir-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:28:43.279Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac", "user": "operator"} 2026-10-05T14:28:43.309Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac", "user": "monitor"} 2026-10-05T14:28:43.353Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac"} 2026-10-05T14:28:43.386Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac"} 2026-10-05T14:28:43.414Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac", "user": "xtrabackup"} 2026-10-05T14:28:43.459Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac"} 2026-10-05T14:28:43.493Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac", "user": "replication"} 2026-10-05T14:28:43.501Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "90ee13f3-6300-457e-977b-789c3bfcf2ac", "err": "get primary pxc pod: not found"} 2026-10-05T14:28:48.606Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a34a301a-bc09-4b34-87af-34436d5ae580", "err": "get primary pxc pod: not found"} 2026-10-05T14:28:53.699Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "fcaced82-3334-4fa3-aea0-7276b13f593d", "err": "get primary pxc pod: not found"} 2026-10-05T14:28:58.796Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "71e5643b-492d-4e44-b649-eaae9131ca65", "err": "get primary pxc pod: not found"} 2026-10-05T14:30:49.152Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "1ab113ec-3f83-4c04-8cbf-43c144ce3128", "user": "root"} 2026-10-05T14:30:49.267Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "1ab113ec-3f83-4c04-8cbf-43c144ce3128", "new version": "8.0.43-34.1"} 2026-10-05T14:30:50.393Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "1ab113ec-3f83-4c04-8cbf-43c144ce3128"} 2026-10-05T14:30:56.910Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "43e062d9-3e43-4a6f-86de-43acce6aaba4"} 2026-10-05T14:31:02.254Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "12a3f3b4-7a3f-4900-bc13-3933f2124534"} 2026-10-05T14:31:07.561Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "141d9b70-c9cf-4c09-b14a-893a1b1eac99"} 2026-10-05T14:31:13.057Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "3a707613-f8e5-40f7-8487-3437534ed55c"} 2026-10-05T14:31:18.044Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a9d2b843-5167-47fb-bf00-1b26cd678485"} 2026-10-05T14:31:23.332Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "c1feb6ba-75aa-4b64-86c8-ac2defb26bd8"} 2026-10-05T14:31:29.149Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "82b263fb-e60d-4c1c-8444-1583eefb8b8a"} 2026-10-05T14:31:34.448Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "7d1ff6c4-7f9b-4eca-992b-20d891bbbd56"} 2026-10-05T14:31:39.401Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "a6a398cc-1dfb-4f58-a2fb-30f659d8779d"} 2026-10-05T14:31:44.896Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "8643e910-ff75-4401-8ecd-7005be40b1e1"} 2026-10-05T14:31:50.059Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "9243dd94-73c7-4be6-a17e-a40b47191e7b"} 2026-10-05T14:31:55.251Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "18ff60e7-a6ee-4ee0-b025-e03f78b5abce"} 2026-10-05T14:32:00.673Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "5c8c84e8-368c-4362-be64-041d24f8fb81"} 2026-10-05T14:32:05.927Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "9f65474c-b191-4fa3-aed1-7f03912aaf48"} 2026-10-05T14:32:11.253Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "aae3e21a-c47d-4e5d-aa7b-8798775d50c6"} 2026-10-05T14:32:16.530Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "faff9c3d-00f6-4602-9124-cbcfde43baab"} 2026-10-05T14:32:21.542Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "bd9fb7ff-2ae8-4574-b204-a845a6321f3c"} 2026-10-05T14:32:27.032Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "f945c29d-8590-4f12-8818-8c53620f3f60"} 2026-10-05T14:32:32.396Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "d9a5786c-b680-4304-9a72-e8d872905bae"} 2026-10-05T14:32:37.711Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "280a6b10-3984-4868-b33e-37a721d1c831"} 2026-10-05T14:32:43.547Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "3ffe1406-567c-4341-b780-78831359c8c2"} 2026-10-05T14:32:48.317Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"emptydir","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "emptydir", "reconcileID": "808c0e3f-799d-44cd-be6f-6a7c909234d7"} 2026-10-05T14:32:59.943Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "version": "1.21.0"} 2026-10-05T14:33:00.139Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "object": "hostpath-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:00.193Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "object": "hostpath-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:00.227Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "object": "hostpath-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:33:00.294Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "object": "hostpath-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:00.363Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dc18a756-47ac-4add-bfdf-b76efacfb243", "object": "hostpath-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-10-05T14:33:01.142Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "beb28792-4e25-4626-a14d-b3257c45d3ea", "object": "hostpath-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-10-05T14:33:01.228Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "beb28792-4e25-4626-a14d-b3257c45d3ea", "object": "hostpath-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:33:06.438Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "654fb57c-9b25-477f-a6cf-c9e904f6f2c5", "object": "hostpath-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-10-05T14:33:37.498Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385", "user": "operator"} 2026-10-05T14:33:37.532Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385", "user": "monitor"} 2026-10-05T14:33:37.563Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385"} 2026-10-05T14:33:37.608Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385"} 2026-10-05T14:33:37.645Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385", "user": "xtrabackup"} 2026-10-05T14:33:37.679Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385"} 2026-10-05T14:33:37.708Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385", "user": "replication"} 2026-10-05T14:33:37.718Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ef262ebc-764c-4238-80d6-273c4f334385", "err": "get primary pxc pod: not found"} 2026-10-05T14:33:42.820Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "fad78fd4-2d23-4487-9636-bc9363f3be3d", "err": "get primary pxc pod: not found"} 2026-10-05T14:33:47.926Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "d5e6cf94-c6cc-4736-8175-477f3376f376", "err": "get primary pxc pod: not found"} 2026-10-05T14:33:53.031Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ec9853cc-d222-4293-9563-f3644fc7dfb3", "err": "get primary pxc pod: not found"} 2026-10-05T14:33:58.174Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "b21c78f6-87d6-41e6-a81e-a72b82fba221", "err": "get primary pxc pod: not found"} 2026-10-05T14:35:47.474Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "2b93b944-762b-433d-b754-c774849af553", "err": "failed to ensure cluster readonly status: connect to pod hostpath-pxc-2: dial tcp: lookup hostpath-pxc-2.hostpath-pxc.storage-14058 on 34.118.224.10:53: no such host"} 2026-10-05T14:35:48.280Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "68f37924-a836-460c-83d4-7ac42016ab4d", "user": "root"} 2026-10-05T14:35:48.333Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "68f37924-a836-460c-83d4-7ac42016ab4d", "err": "failed to ensure cluster readonly status: connect to pod hostpath-pxc-2: dial tcp: lookup hostpath-pxc-2.hostpath-pxc.storage-14058 on 34.118.224.10:53: no such host"} 2026-10-05T14:35:48.367Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "68f37924-a836-460c-83d4-7ac42016ab4d", "new version": "8.0.43-34.1"} 2026-10-05T14:35:48.976Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "6cb31627-b892-42b9-8ab8-a045fde3e7ba", "err": "failed to ensure cluster readonly status: connect to pod hostpath-pxc-2: dial tcp: lookup hostpath-pxc-2.hostpath-pxc.storage-14058 on 34.118.224.10:53: no such host"} 2026-10-05T14:35:49.683Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "68f37924-a836-460c-83d4-7ac42016ab4d"} 2026-10-05T14:35:56.084Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "40442bcc-0487-4632-81a5-4ca480c306f7"} 2026-10-05T14:36:01.525Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "5c512303-8fa4-4391-80a9-cd3afd0d7e9e"} 2026-10-05T14:36:06.724Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "66ee624f-104a-450c-9f83-3ae09a32392b"} 2026-10-05T14:36:11.792Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "dd33dccc-6e73-4376-8f85-8bb32afee846"} 2026-10-05T14:36:17.489Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "cb89e1fe-38d6-4d29-a473-65fcf76f163b"} 2026-10-05T14:36:22.801Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "e6182578-2fc9-4815-b3e3-e7cc33846f04"} 2026-10-05T14:36:28.487Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "748131db-b606-4e8b-9857-92f0efc20789"} 2026-10-05T14:36:33.252Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "fc482bb8-9b0a-4710-8534-0f720a990060"} 2026-10-05T14:36:38.282Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "6b2c59b9-4935-4c55-b26b-25d61f652dd5"} 2026-10-05T14:36:43.928Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "12375b38-e587-42a2-9a7f-3636c3afb594"} 2026-10-05T14:36:49.156Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "d08b10b6-3bb6-41b4-ac5b-7c2c6c418ba2"} 2026-10-05T14:36:54.414Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "7a8e6119-f664-4b45-8496-d622b7059042"} 2026-10-05T14:36:59.825Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "6718b30a-18bb-47de-b569-804d07893682"} 2026-10-05T14:37:05.110Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "2ce5cc6f-331d-42eb-8196-9d3103d8c192"} 2026-10-05T14:37:10.369Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "e7ab61d8-f4b2-48c8-801a-fc4993814156"} 2026-10-05T14:37:15.750Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "99aaf1e3-06b8-4070-bf55-0145e679c446"} 2026-10-05T14:37:20.928Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "3f57becb-8b7f-4547-b2f3-2eea7f2e91db"} 2026-10-05T14:37:25.991Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "c99facba-5e9c-4de9-821e-dcc46c19063f"} 2026-10-05T14:37:31.554Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "71b9a7f4-554d-455e-8228-b5dde20fc6ab"} 2026-10-05T14:37:36.746Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "553756c8-5c89-44be-adf1-b84b76776d77"} 2026-10-05T14:37:42.211Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "e0d42e5a-0c22-477c-bece-0e933f756b23"} 2026-10-05T14:37:47.453Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "9e636c80-a285-468f-958e-fe2947992666"} 2026-10-05T14:37:52.789Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "03c42894-f1c1-4a30-98d7-8b8bbaf356af"} 2026-10-05T14:37:59.003Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "4bbecee4-8946-4f59-9de5-f1c0031cab40"} 2026-10-05T14:38:04.301Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ec2c581a-d069-4f8b-b56b-bb6fed849015"} 2026-10-05T14:38:09.571Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "50867e97-0322-4d70-8763-6e9e6b98932e"} 2026-10-05T14:38:14.818Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "89102623-4b39-4682-b7c9-cd231b168762"} 2026-10-05T14:38:20.052Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "e708f32e-b2d0-4ccc-a9fe-87d59db15208"} 2026-10-05T14:38:25.334Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "6f2f8211-8739-45ce-95df-150e6c149316"} 2026-10-05T14:38:30.658Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ebd605db-674b-4257-a59b-ccdc0fb6478e"} 2026-10-05T14:38:35.944Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "8593af88-e034-4865-b9cd-581abb8f07d5"} 2026-10-05T14:38:41.325Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "1aa3ed38-5641-49d0-b35b-852748da30d0"} 2026-10-05T14:38:46.717Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "51b3643e-4d63-4761-b9f5-99955424d203"} 2026-10-05T14:38:51.955Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "88c8d72b-82e8-4b68-9049-3e8393f78682"} 2026-10-05T14:38:57.289Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "ebe7dbaf-05ae-473e-9110-4613b6ed6d11"} 2026-10-05T14:39:02.531Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "df345960-3dda-449b-847f-bc5b04c3432e"} 2026-10-05T14:39:07.853Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "4238e28f-b6e2-4072-889c-dcd6ef8f6fb9"} 2026-10-05T14:39:13.544Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "e90cea4a-455f-49c1-a508-2e6c33c71953"} 2026-10-05T14:39:18.444Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "33c2038b-91b7-4371-906c-488df1200341"} 2026-10-05T14:39:23.714Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "48b5201b-6dfb-47fe-b90f-51fe5fd5c20c"} 2026-10-05T14:39:28.921Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "b874b3a1-025e-4534-b567-62e2255e5bb7"} 2026-10-05T14:39:34.256Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "552c3b8f-aa34-466f-8d1a-bda845ea7aa5"} 2026-10-05T14:39:42.924Z ERROR sync users {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"hostpath","namespace":"storage-14058"}, "namespace": "storage-14058", "name": "hostpath", "reconcileID": "688ae7e2-dcfe-450e-a9b6-01a2c1cc0b95", "error": "exec syncusers: failed to execute command in pod: unable to upgrade connection: pods \"hostpath-proxysql-1\" not found / / ", "errorVerbose": "exec syncusers: failed to execute command in pod: unable to upgrade connection: pods \"hostpath-proxysql-1\" not found / / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:1000\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:859\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1771"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1 + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names= + [[ -n '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.l2GPdupwUf ++ mktemp + local LAST_ERR=/tmp/tmp.tKRZMtHJJR + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l2GPdupwUf validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.tKRZMtHJJR + rm /tmp/tmp.l2GPdupwUf /tmp/tmp.tKRZMtHJJR + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace storage-14058 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ date +%s ++ mktemp + local deadline=1791211357 ++ mktemp + local LAST_OUT=/tmp/tmp.tDQdh9CKEi + local LAST_OUT=/tmp/tmp.K7ksYRuzzl ++ mktemp ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + local LAST_ERR=/tmp/tmp.H8JOCSwvqh + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.YZUxWNabD5 + local exit_status=0 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace storage-14058 ++ seq 0 2 + for i in $(seq 0 2) + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211242 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211251 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.K7ksYRuzzl namespace "pxc-operator" force deleted + cat /tmp/tmp.YZUxWNabD5 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.K7ksYRuzzl /tmp/tmp.YZUxWNabD5 + return 0 + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211260 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211270 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211278 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + [[ -z namespace/storage-14058 ]] ++ date +%s + [[ 1791211287 -ge 1791211357 ]] + sleep 5 ++ kubectl_bin get namespace storage-14058 --ignore-not-found -o name + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tDQdh9CKEi namespace "storage-14058" force deleted + cat /tmp/tmp.H8JOCSwvqh Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.tDQdh9CKEi /tmp/tmp.H8JOCSwvqh + return 0 + [[ -z '' ]] ++ date +%s + deadline=1791211415 ++ awk '$2 != ""' ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' + [[ -z '' ]] + rm -rf /tmp/tmp.E6q2wqkgYo + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------