Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/scaling-8-0.log Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1 + create_infra scaling-28830 + local ns=scaling-28830 + '[' -n pxc-operator ']' + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + read -r pns name + [[ -z scaling-9128 ]] + [[ -z scaling ]] + kubectl_bin patch pxc -n scaling-9128 scaling --type=merge -p '{"metadata":{"finalizers":[]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.LMW6VqMbyp ++ mktemp + local LAST_ERR=/tmp/tmp.6hvfunbyp0 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc -n scaling-9128 scaling --type=merge -p '{"metadata":{"finalizers":[]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LMW6VqMbyp perconaxtradbcluster.pxc.percona.com/scaling patched (no change) + cat /tmp/tmp.6hvfunbyp0 + rm /tmp/tmp.LMW6VqMbyp /tmp/tmp.6hvfunbyp0 + return 0 + read -r pns name + read -r pns name + kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found perconaxtradbcluster.pxc.percona.com "scaling" deleted from scaling-9128 namespace + timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found No resources found + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ awk '-F ' '{print $2}' ++ tail -n1 ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// + local chaos_mesh_ns= + '[' -n '' ']' ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep validate-auth ++ kubectl get ValidatingWebhookConfiguration + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ grep chaos-mesh.org ++ awk '{print $1}' ++ kubectl get crd + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ grep chaos-mesh ++ kubectl get clusterrole + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns + awk '{print$1}' + kubectl_bin get ns ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + local LAST_OUT=/tmp/tmp.HzD92YqlUl ++ mktemp + local LAST_ERR=/tmp/tmp.kOCQOV9o32 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.6Ji5LXTe6L ++ mktemp + local LAST_ERR=/tmp/tmp.QbYKBseAyX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6Ji5LXTe6L + cat /tmp/tmp.QbYKBseAyX + rm /tmp/tmp.6Ji5LXTe6L /tmp/tmp.QbYKBseAyX + return 0 namespace "scaling-9128" deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HzD92YqlUl namespace "pxc-operator" deleted + cat /tmp/tmp.kOCQOV9o32 + rm /tmp/tmp.HzD92YqlUl /tmp/tmp.kOCQOV9o32 + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'waiting for namespace/pxc-operator to be deleted' waiting for namespace/pxc-operator to be deleted+ set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.gsQ4ZUEqm5 ++ mktemp + local LAST_ERR=/tmp/tmp.hSJcl8o1Yl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gsQ4ZUEqm5 namespace/pxc-operator created + cat /tmp/tmp.hSJcl8o1Yl + rm /tmp/tmp.gsQ4ZUEqm5 /tmp/tmp.hSJcl8o1Yl + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.2SNinogAEm +++ mktemp ++ local LAST_ERR=/tmp/tmp.mMnpMs0OsP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2SNinogAEm ++ cat /tmp/tmp.mMnpMs0OsP ++ rm /tmp/tmp.2SNinogAEm /tmp/tmp.mMnpMs0OsP ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.gceOI3BQNf ++ mktemp + local LAST_ERR=/tmp/tmp.jX7h1ASLBX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gceOI3BQNf Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4" modified. + cat /tmp/tmp.jX7h1ASLBX + rm /tmp/tmp.gceOI3BQNf /tmp/tmp.jX7h1ASLBX + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.iyG6mvyjE3 ++ mktemp + local LAST_ERR=/tmp/tmp.Gna67O0y4l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.iyG6mvyjE3 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.Gna67O0y4l + rm /tmp/tmp.iyG6mvyjE3 /tmp/tmp.Gna67O0y4l + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.PMYxa3bKiO ++ mktemp + local LAST_ERR=/tmp/tmp.s4a5MXFOKC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.PMYxa3bKiO customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met + cat /tmp/tmp.s4a5MXFOKC E0702 04:08:35.490473 11788 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782965311193297000&timeoutSeconds=507&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.PMYxa3bKiO /tmp/tmp.s4a5MXFOKC + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.FVTf3OH4O6 ++ mktemp + local LAST_ERR=/tmp/tmp.3vYJw3UPlT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.FVTf3OH4O6 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met + cat /tmp/tmp.3vYJw3UPlT E0702 04:08:37.315553 12058 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782965316193620000&timeoutSeconds=557&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.FVTf3OH4O6 /tmp/tmp.3vYJw3UPlT + return 0 + kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com ++ mktemp + local LAST_OUT=/tmp/tmp.4oL5P1TkH6 ++ mktemp + local LAST_ERR=/tmp/tmp.hIgdY07KCj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4oL5P1TkH6 customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met + cat /tmp/tmp.hIgdY07KCj E0702 04:08:39.249600 12310 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782965316193620000&timeoutSeconds=328&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.4oL5P1TkH6 /tmp/tmp.hIgdY07KCj + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + sed -e 's^namespace: .*^namespace: pxc-operator^' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.HYKRn2W1dG ++ mktemp + local LAST_ERR=/tmp/tmp.dVVdC2otlg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HYKRn2W1dG clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.dVVdC2otlg + rm /tmp/tmp.HYKRn2W1dG /tmp/tmp.dVVdC2otlg + return 0 + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' - + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.23ILKbhinE ++ mktemp + local LAST_ERR=/tmp/tmp.l3BzsF44K4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.23ILKbhinE deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.l3BzsF44K4 + rm /tmp/tmp.23ILKbhinE /tmp/tmp.l3BzsF44K4 + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.lxUkaGKNGR ++ mktemp + local LAST_ERR=/tmp/tmp.JUbVXqZpcQ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lxUkaGKNGR pod/percona-xtradb-cluster-operator-65b5d85575-zcc5r condition met + cat /tmp/tmp.JUbVXqZpcQ E0702 04:08:55.857836 14141 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-zcc5r&resourceVersion=1782965335497360000&timeoutSeconds=359&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" + rm /tmp/tmp.lxUkaGKNGR /tmp/tmp.JUbVXqZpcQ + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ grep -c percona-xtradb-cluster-operator +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ head -1 +++ mktemp ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ local LAST_OUT=/tmp/tmp.FHLwKMc062 +++ mktemp ++ local LAST_ERR=/tmp/tmp.2vvgCU6mdF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.FHLwKMc062 ++ cat /tmp/tmp.2vvgCU6mdF ++ rm /tmp/tmp.FHLwKMc062 /tmp/tmp.2vvgCU6mdF ++ return 0 + wait_pod percona-xtradb-cluster-operator-65b5d85575-zcc5r 480 pxc-operator + local pod=percona-xtradb-cluster-operator-65b5d85575-zcc5r + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-65b5d85575-zcc5r ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-65b5d85575-zcc5r condition met E0702 04:08:59.125438 14619 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-zcc5r&resourceVersion=1782965338352512000&timeoutSeconds=310&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/percona-xtradb-cluster-operator-65b5d85575-zcc5r to become Ready.Ok + sleep 3 + create_namespace scaling-28830 + local namespace=scaling-28830 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ grep chaos-mesh ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ awk '{print $1}' ++ grep validate-auth + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ awk '{print $1}' ++ grep chaos-mesh ++ grep chaos-mesh.org ++ kubectl get crd ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get clusterrolebinding + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces scaling-28830' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces scaling-28830 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace scaling-28830 + awk '{print$1}' ++ mktemp + grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME' + kubectl_bin get ns + local LAST_OUT=/tmp/tmp.DJOLMu07uk + xargs kubectl delete ns ++ mktemp + local LAST_OUT=/tmp/tmp.o8y1eRwhHW ++ mktemp + local LAST_ERR=/tmp/tmp.0uhyUWSpNk + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.Nxuc8BMP3y + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scaling-28830 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scaling-28830 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.o8y1eRwhHW + cat /tmp/tmp.Nxuc8BMP3y + rm /tmp/tmp.o8y1eRwhHW /tmp/tmp.Nxuc8BMP3y + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace scaling-28830 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.DJOLMu07uk + cat /tmp/tmp.0uhyUWSpNk Error from server (NotFound): namespaces "scaling-28830" not found + rm /tmp/tmp.DJOLMu07uk /tmp/tmp.0uhyUWSpNk + return 1 + : + wait_for_delete namespace/scaling-28830 + local res=namespace/scaling-28830 + echo -n 'waiting for namespace/scaling-28830 to be deleted' waiting for namespace/scaling-28830 to be deleted+ set +o xtrace Error from server (NotFound): namespaces "scaling-28830" not found + desc 'create namespace scaling-28830' + set +o xtrace ----------------------------------------------------------------------------------- create namespace scaling-28830 ----------------------------------------------------------------------------------- + kubectl_bin create namespace scaling-28830 ++ mktemp + local LAST_OUT=/tmp/tmp.zac5ozdAro ++ mktemp + local LAST_ERR=/tmp/tmp.esLB2zzRMZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace scaling-28830 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zac5ozdAro namespace/scaling-28830 created + cat /tmp/tmp.esLB2zzRMZ + rm /tmp/tmp.zac5ozdAro /tmp/tmp.esLB2zzRMZ + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.CE3g8xlCoR +++ mktemp ++ local LAST_ERR=/tmp/tmp.IKJZ8mTMEX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CE3g8xlCoR ++ cat /tmp/tmp.IKJZ8mTMEX ++ rm /tmp/tmp.CE3g8xlCoR /tmp/tmp.IKJZ8mTMEX ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4 --namespace=scaling-28830 ++ mktemp + local LAST_OUT=/tmp/tmp.9qZUiaIZ5p ++ mktemp + local LAST_ERR=/tmp/tmp.aXiIxqzzoo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4 --namespace=scaling-28830 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9qZUiaIZ5p Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster4" modified. + cat /tmp/tmp.aXiIxqzzoo + rm /tmp/tmp.9qZUiaIZ5p /tmp/tmp.aXiIxqzzoo + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.EMVEcCeUqv ++ mktemp + local LAST_ERR=/tmp/tmp.hJJUcl0rBb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EMVEcCeUqv secret/minio-secret created secret/aws-s3-secret created secret/do-spaces-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.hJJUcl0rBb + rm /tmp/tmp.EMVEcCeUqv /tmp/tmp.hJJUcl0rBb + return 0 + spinup_pxc scaling /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local cluster=scaling + local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.VZ61t68BEt ++ mktemp + local LAST_ERR=/tmp/tmp.FNVHeTjXx2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.VZ61t68BEt secret/my-cluster-secrets created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.FNVHeTjXx2 + rm /tmp/tmp.VZ61t68BEt /tmp/tmp.FNVHeTjXx2 + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.vM6lO0afEK + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + local LAST_ERR=/tmp/tmp.uxt9Pb2d50 + local exit_status=0 + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-28830~ + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vM6lO0afEK deployment.apps/pxc-client created + cat /tmp/tmp.uxt9Pb2d50 + rm /tmp/tmp.vM6lO0afEK /tmp/tmp.uxt9Pb2d50 + return 0 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local pvc_name= + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-28830~ + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ mktemp + local LAST_OUT=/tmp/tmp.H9GFALFTBE + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml ++ mktemp + local LAST_ERR=/tmp/tmp.OpvBlvp1s6 + local exit_status=0 + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' ++ seq 0 2 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.H9GFALFTBE perconaxtradbcluster.pxc.percona.com/scaling created + cat /tmp/tmp.OpvBlvp1s6 + rm /tmp/tmp.H9GFALFTBE /tmp/tmp.OpvBlvp1s6 + return 0 + desc 'waiting pods to be running for scaling' + set +o xtrace ----------------------------------------------------------------------------------- waiting pods to be running for scaling ----------------------------------------------------------------------------------- ++ get_proxy scaling ++ local target_cluster=scaling +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Cd4yoK82zz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.eSRDx2rGyU +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Cd4yoK82zz +++ cat /tmp/tmp.eSRDx2rGyU +++ rm /tmp/tmp.Cd4yoK82zz /tmp/tmp.eSRDx2rGyU +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QsIMktSSEA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iCTtqnIdbK +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QsIMktSSEA +++ cat /tmp/tmp.iCTtqnIdbK +++ rm /tmp/tmp.QsIMktSSEA /tmp/tmp.iCTtqnIdbK +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scaling-proxysql ++ return + local proxy=scaling-proxysql ++ get_proxy_size scaling ++ local cluster=scaling ++ local size=0 +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NPDfn1LIHN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ViA17atGRX +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NPDfn1LIHN +++ cat /tmp/tmp.ViA17atGRX +++ rm /tmp/tmp.NPDfn1LIHN /tmp/tmp.ViA17atGRX +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gahN1v8B5E ++++ mktemp +++ local LAST_ERR=/tmp/tmp.f2PbpaMcKL +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.gahN1v8B5E +++ cat /tmp/tmp.f2PbpaMcKL +++ rm /tmp/tmp.gahN1v8B5E /tmp/tmp.f2PbpaMcKL +++ return 0 ++ [[ true == \t\r\u\e ]] +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PfQEC2hS9q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tgysF8FHtd +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.PfQEC2hS9q +++ cat /tmp/tmp.tgysF8FHtd +++ rm /tmp/tmp.PfQEC2hS9q /tmp/tmp.tgysF8FHtd +++ return 0 ++ size=2 ++ echo 2 + local proxy_size=2 + wait_for_running scaling-pxc 3 + local name=scaling-pxc + local size=3 + local last_pod + local max_retry=480 + [[ -z scaling-pxc ]] + [[ 3 -le 0 ]] + last_pod=2 + desc 'Waiting for 3 running pods for scaling-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 3 running pods for scaling-pxc ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-0 480 + local pod=scaling-pxc-0 + local max_retry=480 + local ns= ++ echo scaling-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scaling-pxc-0 condition met waiting for pod/scaling-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-1 480 + local pod=scaling-pxc-1 + local max_retry=480 + local ns= ++ echo scaling-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scaling-pxc-1 condition met waiting for pod/scaling-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-2 480 + local pod=scaling-pxc-2 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scaling-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scaling-pxc-2 condition met waiting for pod/scaling-pxc-2 to become Ready.Ok + wait_for_running scaling-proxysql 2 + local name=scaling-proxysql + local size=2 + local last_pod + local max_retry=480 + [[ -z scaling-proxysql ]] + [[ 2 -le 0 ]] + last_pod=1 + desc 'Waiting for 2 running pods for scaling-proxysql' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 2 running pods for scaling-proxysql ----------------------------------------------------------------------------------- ++ seq 0 1 + for i in '$(seq 0 $last_pod)' + wait_pod scaling-proxysql-0 480 + local pod=scaling-proxysql-0 + local max_retry=480 + local ns= ++ echo scaling-proxysql-0 ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=proxysql + set +o xtrace pod/scaling-proxysql-0 condition met E0702 04:13:51.802085 21948 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscaling-proxysql-0&resourceVersion=1782965628985158000&timeoutSeconds=335&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scaling-proxysql-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-proxysql-1 480 + local pod=scaling-proxysql-1 + local max_retry=480 + local ns= ++ echo scaling-proxysql-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=proxysql + set +o xtrace pod/scaling-proxysql-1 condition met E0702 04:13:58.759313 23031 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscaling-proxysql-1&resourceVersion=1782965633985172000&timeoutSeconds=527&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scaling-proxysql-1 to become Ready.Ok + wait_cluster_consistency scaling 3 2 + local cluster_name=scaling + local pxc_size=3 + local proxy_size=2 + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local max_seconds=1500 + local delay=5 + local retries=300 + local pxc_ready + local proxy_ready + sleep 7 + echo -n 'waiting for pxc/scaling to be ready' waiting for pxc/scaling to be ready++ seq 1 300 + for _ in '$(seq 1 "${retries}")' ++ kubectl_bin get pxc scaling -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.evFnwUZRbY +++ mktemp ++ local LAST_ERR=/tmp/tmp.jWfURBHZES ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc scaling -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.evFnwUZRbY ++ cat /tmp/tmp.jWfURBHZES ++ rm /tmp/tmp.evFnwUZRbY /tmp/tmp.jWfURBHZES ++ return 0 + state=ready ++ kubectl_bin get pxc scaling -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eoi4IOPg0J +++ mktemp ++ local LAST_ERR=/tmp/tmp.yj264ye9Me ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc scaling -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.eoi4IOPg0J ++ cat /tmp/tmp.yj264ye9Me ++ rm /tmp/tmp.eoi4IOPg0J /tmp/tmp.yj264ye9Me ++ return 0 + pxc_ready=3 + proxy_ready=0 + [[ 2 -gt 0 ]] +++ get_proxy_engine scaling +++ local cluster_name=scaling ++++ get_proxy scaling ++++ local target_cluster=scaling +++++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.WAMppJ08mv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.KaargmJR36 +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.WAMppJ08mv +++++ cat /tmp/tmp.KaargmJR36 +++++ rm /tmp/tmp.WAMppJ08mv /tmp/tmp.KaargmJR36 +++++ return 0 ++++ [[ '' == \t\r\u\e ]] +++++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.O3hoxWaSP3 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.c5FU9pBMJt +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.O3hoxWaSP3 +++++ cat /tmp/tmp.c5FU9pBMJt +++++ rm /tmp/tmp.O3hoxWaSP3 /tmp/tmp.c5FU9pBMJt +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo scaling-proxysql ++++ return +++ local cluster_proxy=scaling-proxysql +++ echo proxysql ++ kubectl_bin get pxc scaling -o 'jsonpath={.status.proxysql.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iGwdW0UaEJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.VDRq7HtCX4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc scaling -o 'jsonpath={.status.proxysql.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iGwdW0UaEJ ++ cat /tmp/tmp.VDRq7HtCX4 ++ rm /tmp/tmp.iGwdW0UaEJ /tmp/tmp.VDRq7HtCX4 ++ return 0 + proxy_ready=2 + [[ ready == \r\e\a\d\y ]] + [[ 3 == \3 ]] + [[ 2 == \2 ]] + echo + return + sleep 10 ++ kubectl get pxc scaling -o 'jsonpath={.spec.secretsName}' + local secret_name=my-cluster-secrets ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode +++ mktemp ++ local LAST_OUT=/tmp/tmp.KXvWoxLEQ7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mpWREEyDkL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.KXvWoxLEQ7 ++ cat /tmp/tmp.mpWREEyDkL ++ rm /tmp/tmp.KXvWoxLEQ7 /tmp/tmp.mpWREEyDkL ++ return 0 + local root_pass=root_password + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scaling-pxc-0.scaling-pxc -uroot -p'\''root_password'\'' -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h scaling-pxc-0.scaling-pxc -uroot -p'\''root_password'\'' -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YTcvDtGHtK +++ mktemp ++ local LAST_ERR=/tmp/tmp.dMB26cfv1q ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.YTcvDtGHtK ++ cat /tmp/tmp.dMB26cfv1q ++ rm /tmp/tmp.YTcvDtGHtK /tmp/tmp.dMB26cfv1q ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' ++ echo pxc-client-67fc4995bb-f7f6h + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:14:32.608930 27857 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965670955468000&timeoutSeconds=476&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + write_data scaling 100500 no-proxy -P3306 + local cluster=scaling + local id=100500 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local root_pass ++ get_password scaling root ++ local cluster=scaling ++ local user=root +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BnJGETc2IU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oaaY5a1yx5 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.BnJGETc2IU +++ cat /tmp/tmp.oaaY5a1yx5 +++ rm /tmp/tmp.BnJGETc2IU /tmp/tmp.oaaY5a1yx5 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ base64 --decode ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SYHlz2UX9J +++ mktemp ++ local LAST_ERR=/tmp/tmp.CUrFMtzxXX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SYHlz2UX9J ++ cat /tmp/tmp.CUrFMtzxXX ++ rm /tmp/tmp.SYHlz2UX9J /tmp/tmp.CUrFMtzxXX ++ return 0 + root_pass=root_password ++ get_proxy scaling ++ local target_cluster=scaling +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.J2pe6CbPAy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.QiX8xQDQtB +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.J2pe6CbPAy +++ cat /tmp/tmp.QiX8xQDQtB +++ rm /tmp/tmp.J2pe6CbPAy /tmp/tmp.QiX8xQDQtB +++ return 0 ++ [[ '' == \t\r\u\e ]] +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A2a8mec2Qd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JJ3dvLGr32 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.A2a8mec2Qd +++ cat /tmp/tmp.JJ3dvLGr32 +++ rm /tmp/tmp.A2a8mec2Qd /tmp/tmp.JJ3dvLGr32 +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo scaling-proxysql ++ return + proxy=scaling-proxysql + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] + host='-h scaling-pxc-0.scaling-pxc' + run_mysql 'INSERT IGNORE myApp.myApp (id) VALUES (100500)' '-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=INSERT IGNORE myApp.myApp (id) VALUES (100500)' + local 'uri=-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hpY6xFctR3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.c9l31Ibz9R ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.hpY6xFctR3 ++ cat /tmp/tmp.c9l31Ibz9R ++ rm /tmp/tmp.hpY6xFctR3 /tmp/tmp.c9l31Ibz9R ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-f7f6h ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:14:47.663000 29944 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965685258185000&timeoutSeconds=342&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + sleep 30 + desc 'compare initial data' + set +o xtrace ----------------------------------------------------------------------------------- compare initial data ----------------------------------------------------------------------------------- + compare_data scaling select-1 no-proxy -P3306 + local cluster=scaling + local compare=select-1 + local use_proxy=no-proxy + local port=-P3306 + local database=myApp + local table=myApp + local size + local root_pass ++ get_password scaling root ++ local cluster=scaling ++ local user=root +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QgUZ2ZqvaL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GIjgbprb76 +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.QgUZ2ZqvaL +++ cat /tmp/tmp.GIjgbprb76 +++ rm /tmp/tmp.QgUZ2ZqvaL /tmp/tmp.GIjgbprb76 +++ return 0 ++ getSecretData my-cluster-secrets root ++ local secretName=my-cluster-secrets ++ local dataKey=root ++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.MzDz8NM8fl +++ mktemp ++ local LAST_ERR=/tmp/tmp.G4a78ND8AY ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}' ++ base64 --decode ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.MzDz8NM8fl ++ cat /tmp/tmp.G4a78ND8AY ++ rm /tmp/tmp.MzDz8NM8fl /tmp/tmp.G4a78ND8AY ++ return 0 + root_pass=root_password + [[ no-proxy == true ]] + [[ no-proxy == \u\s\e\-\p\r\o\x\y ]] ++ get_pxc_size scaling ++ local cluster=scaling ++ local size=0 +++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.pxc.size}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.L1mFgjzCGa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.85rrjdO7DS +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc scaling -o 'jsonpath={.spec.pxc.size}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.L1mFgjzCGa +++ cat /tmp/tmp.85rrjdO7DS +++ rm /tmp/tmp.L1mFgjzCGa /tmp/tmp.85rrjdO7DS +++ return 0 ++ size=3 ++ echo 3 + size=3 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-0.scaling-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LujEBLSAp4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.O0j9c1oj2z ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.LujEBLSAp4 ++ cat /tmp/tmp.O0j9c1oj2z ++ rm /tmp/tmp.LujEBLSAp4 /tmp/tmp.O0j9c1oj2z ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-f7f6h + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:15:35.258650 4250 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965732859357000&timeoutSeconds=594&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.MhYBIbMudc/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.MhYBIbMudc/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.MhYBIbMudc/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-1.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-1.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-1.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-1.scaling-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VIkx2QHmYS +++ mktemp ++ local LAST_ERR=/tmp/tmp.UmXSaIMt6Y ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.VIkx2QHmYS ++ cat /tmp/tmp.UmXSaIMt6Y ++ rm /tmp/tmp.VIkx2QHmYS /tmp/tmp.UmXSaIMt6Y ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ echo pxc-client-67fc4995bb-f7f6h ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:15:46.849474 6009 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965744861157000&timeoutSeconds=543&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.MhYBIbMudc/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.MhYBIbMudc/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.MhYBIbMudc/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-2.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local command_id=select-1 + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-2.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT id FROM myApp.myApp ORDER BY id;' '-h scaling-pxc-2.scaling-pxc -P3306 -uroot -p'\''root_password'\''' + local 'command=SELECT id FROM myApp.myApp ORDER BY id;' + local 'uri=-h scaling-pxc-2.scaling-pxc -P3306 -uroot -p'\''root_password'\''' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OtCLIqMa4G +++ mktemp ++ local LAST_ERR=/tmp/tmp.M7R8MwBJxp ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.OtCLIqMa4G ++ cat /tmp/tmp.M7R8MwBJxp ++ rm /tmp/tmp.OtCLIqMa4G /tmp/tmp.M7R8MwBJxp ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-f7f6h + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:15:58.869882 7700 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965756595016000&timeoutSeconds=431&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.MhYBIbMudc/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.MhYBIbMudc/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.MhYBIbMudc/select-1.sql + is_keyring_plugin_in_use scaling + local cluster=scaling + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + grep -E -o 'early-plugin-load=keyring_\w+.so' + kubectl exec scaling-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' + return 1 + desc 'scale up from 3 to 5' + set +o xtrace ----------------------------------------------------------------------------------- scale up from 3 to 5 ----------------------------------------------------------------------------------- + sed -e 's/size: 3/size: 5/' + kubectl_bin apply -f- + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local pvc_name= + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' ++ mktemp + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-28830~ + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + local LAST_OUT=/tmp/tmp.rMttUWw7FH ++ mktemp + local LAST_ERR=/tmp/tmp.zuaf9nLUsY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f- + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rMttUWw7FH perconaxtradbcluster.pxc.percona.com/scaling configured + cat /tmp/tmp.zuaf9nLUsY + rm /tmp/tmp.rMttUWw7FH /tmp/tmp.zuaf9nLUsY + return 0 + desc 'check if all 5 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 5 Pods started ----------------------------------------------------------------------------------- + wait_for_running scaling-pxc 5 + local name=scaling-pxc + local size=5 + local last_pod + local max_retry=480 + [[ -z scaling-pxc ]] + [[ 5 -le 0 ]] + last_pod=4 + desc 'Waiting for 5 running pods for scaling-pxc' + set +o xtrace ----------------------------------------------------------------------------------- Waiting for 5 running pods for scaling-pxc ----------------------------------------------------------------------------------- ++ seq 0 4 + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-0 480 + local pod=scaling-pxc-0 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scaling-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scaling-pxc-0 condition met E0702 04:16:17.636128 10429 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscaling-pxc-0&resourceVersion=1782965775310239024&timeoutSeconds=585&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scaling-pxc-0 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-1 480 + local pod=scaling-pxc-1 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ echo scaling-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/scaling-pxc-1 condition met E0702 04:16:27.763339 11899 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscaling-pxc-1&resourceVersion=1782965784485020000&timeoutSeconds=427&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scaling-pxc-1 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-2 480 + local pod=scaling-pxc-2 + local max_retry=480 + local ns= ++ echo scaling-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scaling-pxc-2 condition met E0702 04:16:38.156034 13580 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dscaling-pxc-2&resourceVersion=1782965795310671024&timeoutSeconds=347&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/scaling-pxc-2 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-3 480 + local pod=scaling-pxc-3 + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo scaling-pxc-3 + local container=pxc + set +o xtrace pod/scaling-pxc-3 condition met waiting for pod/scaling-pxc-3 to become Ready.Ok + for i in '$(seq 0 $last_pod)' + wait_pod scaling-pxc-4 480 + local pod=scaling-pxc-4 + local max_retry=480 + local ns= ++ echo scaling-pxc-4 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ grep -E '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/scaling-pxc-4 condition met waiting for pod/scaling-pxc-4 to become Ready.Ok + sleep 15 + desc 'check if PVC created' + set +o xtrace ----------------------------------------------------------------------------------- check if PVC created ----------------------------------------------------------------------------------- + compare_kubectl pvc/datadir-scaling-pxc-3 + local resource=pvc/datadir-scaling-pxc-3 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml + local new_result=/tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-3.yml + desc 'compare pvc/datadir-scaling-pxc-3-' + set +o xtrace ----------------------------------------------------------------------------------- compare pvc/datadir-scaling-pxc-3- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.33 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k127.yml ']' + version_gt 1.24 ++ echo '1.33 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k124.yml ']' + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k122.yml ']' + version_gt 1.21 ++ echo '1.33 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-oc.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("scaling-28830", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml pvc/datadir-scaling-pxc-3 ++ mktemp + local LAST_OUT=/tmp/tmp.3N8MPynrIh ++ mktemp + local LAST_ERR=/tmp/tmp.q6yUSgT943 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml pvc/datadir-scaling-pxc-3 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3N8MPynrIh + cat /tmp/tmp.q6yUSgT943 + rm /tmp/tmp.3N8MPynrIh /tmp/tmp.q6yUSgT943 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml /tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-3.yml + log 'compare_kubectl: pvc/datadir-scaling-pxc-3 OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:28+0000]' compare_kubectl: pvc/datadir-scaling-pxc-3 OK [2026-07-02T04:19:28+0000] compare_kubectl: pvc/datadir-scaling-pxc-3 OK + compare_kubectl pvc/datadir-scaling-pxc-4 + local resource=pvc/datadir-scaling-pxc-4 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml + local new_result=/tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-4.yml + desc 'compare pvc/datadir-scaling-pxc-4-' + set +o xtrace ----------------------------------------------------------------------------------- compare pvc/datadir-scaling-pxc-4- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ echo '1.33 >= 1.33' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k127.yml ']' + version_gt 1.24 ++ echo '1.33 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k124.yml ']' + version_gt 1.22 ++ bc -l ++ echo '1.33 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k122.yml ']' + version_gt 1.21 ++ bc -l ++ echo '1.33 >= 1.21' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-oc.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("scaling-28830", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml pvc/datadir-scaling-pxc-4 ++ mktemp + local LAST_OUT=/tmp/tmp.l0AofCiNen ++ mktemp + local LAST_ERR=/tmp/tmp.TNZ0tCJvoR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml pvc/datadir-scaling-pxc-4 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.l0AofCiNen + cat /tmp/tmp.TNZ0tCJvoR + rm /tmp/tmp.l0AofCiNen /tmp/tmp.TNZ0tCJvoR + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml /tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-4.yml + log 'compare_kubectl: pvc/datadir-scaling-pxc-4 OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:19:30+0000]' compare_kubectl: pvc/datadir-scaling-pxc-4 OK [2026-07-02T04:19:30+0000] compare_kubectl: pvc/datadir-scaling-pxc-4 OK + desc 'check data consistency for new Pods' + set +o xtrace ----------------------------------------------------------------------------------- check data consistency for new Pods ----------------------------------------------------------------------------------- + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-3.scaling-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scaling-pxc-3.scaling-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-3.scaling-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scaling-pxc-3.scaling-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EpmiVmFhAh +++ mktemp ++ local LAST_ERR=/tmp/tmp.sw8b7tEat5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.EpmiVmFhAh ++ cat /tmp/tmp.sw8b7tEat5 ++ rm /tmp/tmp.EpmiVmFhAh /tmp/tmp.sw8b7tEat5 ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-f7f6h ++ grep -E '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:19:35.650437 8312 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965973318980000&timeoutSeconds=409&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.MhYBIbMudc/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.MhYBIbMudc/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.MhYBIbMudc/select-1.sql + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-4.scaling-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scaling-pxc-4.scaling-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1-80.sql ]] + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]] + run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-4.scaling-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h scaling-pxc-4.scaling-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2ga0bJCaQC +++ mktemp ++ local LAST_ERR=/tmp/tmp.SY8yCsA58H ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.2ga0bJCaQC ++ cat /tmp/tmp.SY8yCsA58H ++ rm /tmp/tmp.2ga0bJCaQC /tmp/tmp.SY8yCsA58H ++ return 0 + client_pod=pxc-client-67fc4995bb-f7f6h + wait_pod pxc-client-67fc4995bb-f7f6h + local pod=pxc-client-67fc4995bb-f7f6h + local max_retry=480 + local ns= ++ grep -E '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-67fc4995bb-f7f6h + local container= + set +o xtrace pod/pxc-client-67fc4995bb-f7f6h condition met E0702 04:19:49.611495 10459 reflector.go:227] "Failed to watch" err="Get \"https://34.172.197.9/api/v1/namespaces/scaling-28830/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpxc-client-67fc4995bb-f7f6h&resourceVersion=1782965987024408000&timeoutSeconds=566&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured" waiting for pod/pxc-client-67fc4995bb-f7f6h to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup .Ok + set +o xtrace + [[ ! -s /tmp/tmp.MhYBIbMudc/select-1.sql ]] ++ grep 'Unknown MySQL server host' /tmp/tmp.MhYBIbMudc/select-1.sql + [[ -n '' ]] + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.MhYBIbMudc/select-1.sql + sleep 15 + desc 'check new Pods exists in ProxySQL' + set +o xtrace ----------------------------------------------------------------------------------- check new Pods exists in ProxySQL ----------------------------------------------------------------------------------- + grep scaling-pxc-3 + run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0 + local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local pod=scaling-proxysql-0 + local container_name= + set +o xtrace scaling-pxc-3.scaling-pxc.scaling-28830.svc.cluster.local scaling-pxc-3.scaling-pxc.scaling-28830.svc.cluster.local + grep scaling-pxc-4 + run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0 + local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local pod=scaling-proxysql-0 + local container_name= + set +o xtrace scaling-pxc-4.scaling-pxc.scaling-28830.svc.cluster.local scaling-pxc-4.scaling-pxc.scaling-28830.svc.cluster.local + desc 'scale down from 5 to 3' + set +o xtrace ----------------------------------------------------------------------------------- scale down from 5 to 3 ----------------------------------------------------------------------------------- + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local pvc_name= + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml '' + local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + local pvc_name= + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/conf/scaling.yml + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-28830~ + /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + local LAST_OUT=/tmp/tmp.O6HfIQHelg ++ mktemp + local LAST_ERR=/tmp/tmp.lKdQ2LazV5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.O6HfIQHelg perconaxtradbcluster.pxc.percona.com/scaling configured + cat /tmp/tmp.lKdQ2LazV5 + rm /tmp/tmp.O6HfIQHelg /tmp/tmp.lKdQ2LazV5 + return 0 + sleep 25 + desc 'check if Pod deleted' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod deleted ----------------------------------------------------------------------------------- + wait_for_delete pod/scaling-pxc-3 + local res=pod/scaling-pxc-3 + echo -n 'waiting for pod/scaling-pxc-3 to be deleted' waiting for pod/scaling-pxc-3 to be deleted+ set +o xtrace ..Error from server (NotFound): pods "scaling-pxc-3" not found + wait_for_delete pod/scaling-pxc-4 + local res=pod/scaling-pxc-4 + echo -n 'waiting for pod/scaling-pxc-4 to be deleted' waiting for pod/scaling-pxc-4 to be deleted+ set +o xtrace Error from server (NotFound): pods "scaling-pxc-4" not found + desc 'check if PVC not deleted' + set +o xtrace ----------------------------------------------------------------------------------- check if PVC not deleted ----------------------------------------------------------------------------------- + compare_kubectl pvc/datadir-scaling-pxc-3 + local resource=pvc/datadir-scaling-pxc-3 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml + local new_result=/tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-3.yml + desc 'compare pvc/datadir-scaling-pxc-3-' + set +o xtrace ----------------------------------------------------------------------------------- compare pvc/datadir-scaling-pxc-3- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.33 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k133.yml ']' + version_gt 1.29 ++ echo '1.33 >= 1.29' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k129.yml ']' + version_gt 1.27 ++ bc -l ++ echo '1.33 >= 1.27' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k127.yml ']' + version_gt 1.24 ++ bc -l ++ echo '1.33 >= 1.24' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k124.yml ']' + version_gt 1.22 ++ echo '1.33 >= 1.22' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k122.yml ']' + version_gt 1.21 ++ bc -l ++ echo '1.33 >= 1.21' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("scaling-28830", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml pvc/datadir-scaling-pxc-3 ++ mktemp + local LAST_OUT=/tmp/tmp.wSwdOVqJDg ++ mktemp + local LAST_ERR=/tmp/tmp.kCVvbGYHhK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml pvc/datadir-scaling-pxc-3 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wSwdOVqJDg + cat /tmp/tmp.kCVvbGYHhK + rm /tmp/tmp.wSwdOVqJDg /tmp/tmp.kCVvbGYHhK + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml /tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-3.yml + log 'compare_kubectl: pvc/datadir-scaling-pxc-3 OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:21:01+0000]' compare_kubectl: pvc/datadir-scaling-pxc-3 OK [2026-07-02T04:21:01+0000] compare_kubectl: pvc/datadir-scaling-pxc-3 OK + compare_kubectl pvc/datadir-scaling-pxc-4 + local resource=pvc/datadir-scaling-pxc-4 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml + local new_result=/tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-4.yml + desc 'compare pvc/datadir-scaling-pxc-4-' + set +o xtrace ----------------------------------------------------------------------------------- compare pvc/datadir-scaling-pxc-4- ----------------------------------------------------------------------------------- + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-80.yml ']' + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]] + version_gt 1.33 ++ bc -l ++ echo '1.33 >= 1.33' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k133.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k129.yml ']' + version_gt 1.27 ++ echo '1.33 >= 1.27' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k127.yml ']' + version_gt 1.24 ++ echo '1.33 >= 1.24' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k124.yml ']' + version_gt 1.22 ++ bc -l ++ echo '1.33 >= 1.22' + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k122.yml ']' + version_gt 1.21 ++ echo '1.33 >= 1.21' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k121.yml ']' + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-oc.yml ']' + version_gt 1.29 ++ bc -l ++ echo '1.33 >= 1.29' + '[' 1 -eq 1 ']' + return 0 + '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k129-oc.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']' + '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-aks.yml ']' + yq eval ' del(.metadata.managedFields) | del(.. | select(has("creationTimestamp")).creationTimestamp) | del(.. | select(has("namespace")).namespace) | del(.. | select(has("uid")).uid) | del(.metadata.resourceVersion) | del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) | del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) | del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) | del(.spec.template.spec.containers[].env[] | select(.name == "XTRABACKUP_ENABLED")) | del(.metadata.selfLink) | del(.metadata.deletionTimestamp) | del(.metadata.labels."batch.kubernetes.io/job-name") | del(.metadata.labels."job-name") | del(.metadata.labels."topology.kubernetes.io/region") | del(.metadata.labels."topology.kubernetes.io/zone") | del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") | del(.metadata.annotations."kubernetes.io/psp") | del(.metadata.annotations."batch.kubernetes.io/job-tracking") | del(.metadata.annotations."networking.gke.io/target-pool") | del(.metadata.annotations."cloud.google.com/neg") | del(.metadata.annotations."k8s.v1.cni.cncf.io*") | del(.metadata.annotations."k8s.ovn.org/pod-networks") | del(.metadata.annotations."kubernetes.digitalocean.com/load-balancer-id") | del(.metadata.annotations."service.beta.kubernetes.io/do-loadbalancer-type") | del(.spec.template.metadata.annotations."last-applied-secret") | del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") | del(.spec.template.metadata.labels."job-name") | del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") | del(.. | select(has("image")).image) | del(.. | select(has("clusterIP")).clusterIP) | del(.. | select(has("clusterIPs")).clusterIPs) | del(.. | select(has("dataSource")).dataSource) | del(.. | select(has("procMount")).procMount) | del(.. | select(has("storageClassName")).storageClassName) | del(.. | select(has("finalizers")).finalizers) | del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") | del(.. | select(has("volumeName")).volumeName) | del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") | del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") | del(.spec.volumeMode) | del(.spec.nodeName) | del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") | del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") | del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") | del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") | del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") | del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") | del(.. | select(has("kubectl.kubernetes.io/default-container"))."kubectl.kubernetes.io/default-container") | del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) | del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) | del(.. | select(has("nodePort")).nodePort) | del(.. | select(has("imagePullSecrets")).imagePullSecrets) | del(.. | select(has("enableServiceLinks")).enableServiceLinks) | del(.status) | del(.spec.volumeClaimTemplates[].apiVersion) | del(.spec.volumeClaimTemplates[].kind) | del(.metadata.ownerReferences[].apiVersion) | del(.. | select(has("controller-uid")).controller-uid) | del(.. | select(has("preemptionPolicy")).preemptionPolicy) | del(.spec.ipFamilies) | del(.spec.ipFamilyPolicy) | del(.spec.updateStrategy.rollingUpdate.maxUnavailable) | (.. | select(. == "policy/v1beta1")) = "policy/v1" | del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") | (.. | select(tag == "!!str")) |= sub("scaling-28830", "namespace") | (.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") | del(.. | select(has("annotations")).annotations | select(length==0)) | del(.spec.crVersion) | del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' - + kubectl_bin get -o yaml pvc/datadir-scaling-pxc-4 ++ mktemp + local LAST_OUT=/tmp/tmp.YPgMlau9IB ++ mktemp + local LAST_ERR=/tmp/tmp.z8D1aE8EF7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get -o yaml pvc/datadir-scaling-pxc-4 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YPgMlau9IB + cat /tmp/tmp.z8D1aE8EF7 + rm /tmp/tmp.YPgMlau9IB /tmp/tmp.z8D1aE8EF7 + return 0 + [[ 0 -eq 0 ]] + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml /tmp/tmp.MhYBIbMudc/pvc_datadir-scaling-pxc-4.yml + log 'compare_kubectl: pvc/datadir-scaling-pxc-4 OK' ++ date +%Y-%m-%dT%H:%M:%S%z + echo '[2026-07-02T04:21:04+0000]' compare_kubectl: pvc/datadir-scaling-pxc-4 OK [2026-07-02T04:21:04+0000] compare_kubectl: pvc/datadir-scaling-pxc-4 OK + desc 'check if Pod deleted from ProxySQL' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod deleted from ProxySQL ----------------------------------------------------------------------------------- + run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0 + grep scaling-pxc-3 + local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local pod=scaling-proxysql-0 + local container_name= + set +o xtrace + : + run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0 + local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' + local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' + local pod=scaling-proxysql-0 + local container_name= + set +o xtrace + grep scaling-pxc-4 + : + destroy scaling-28830 + local namespace=scaling-28830 + local ignore_logs=true + [[ 0 == 1 ]] + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + tee /tmp/tmp.MhYBIbMudc/operator.log + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'get backup status: Job.batch' + sort -u + grep -v level=info + grep -v 'the object has been modified' ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ head -1 ++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name' ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.XOkcR5W1Ir +++ mktemp ++ local LAST_ERR=/tmp/tmp.YBYQkjSbYX ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.XOkcR5W1Ir ++ cat /tmp/tmp.YBYQkjSbYX ++ rm /tmp/tmp.XOkcR5W1Ir /tmp/tmp.YBYQkjSbYX ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-65b5d85575-zcc5r ++ mktemp + local LAST_OUT=/tmp/tmp.yDnPWbzbQN ++ mktemp + local LAST_ERR=/tmp/tmp.HIWkXjrtmo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-65b5d85575-zcc5r + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yDnPWbzbQN + cat /tmp/tmp.HIWkXjrtmo + rm /tmp/tmp.yDnPWbzbQN /tmp/tmp.HIWkXjrtmo + return 0 2026-07-02T04:08:45.132Z INFO setup Feature gates {"PXCO_FEATURE_GATES": "", "enabled": ""} 2026-07-02T04:08:45.132Z INFO setup Manager starting up {"gitCommit": "dcdf1e8aedba334999b2bd5fa9efcdb1f10e79e0", "gitBranch": "PR-2539-dcdf1e8a", "buildTime": "2026-07-02T01:14:56Z", "goVersion": "go1.26.4", "os": "linux", "arch": "amd64"} 2026-07-02T04:08:45.132Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.33.12-gke.1059000"} 2026-07-02T04:08:45.135Z INFO setup Registering Components. 2026-07-02T04:08:45.961Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2026-07-02T04:08:45.961Z INFO controller-runtime.metrics Starting metrics server 2026-07-02T04:08:45.961Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2026-07-02T04:08:45.961Z INFO setup Starting the Cmd. 2026-07-02T04:08:45.961Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2026-07-02T04:08:45.962Z INFO controller-runtime.certwatcher Starting certificate poll+watcher {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"} 2026-07-02T04:08:45.962Z INFO controller-runtime.certwatcher Updated current TLS certificate {"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"} 2026-07-02T04:08:45.962Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2026-07-02T04:08:45.962Z INFO controller-runtime.webhook Starting webhook server 2026-07-02T04:08:46.063Z INFO Attempting to acquire leader lease... {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:08:46.097Z DEBUG events percona-xtradb-cluster-operator-65b5d85575-zcc5r_81f16a02-5082-4bd5-af32-181af1e33fa1 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"c130e38e-eb9a-4ace-b270-0dd903054b51","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1782965326090303009"}, "reason": "LeaderElection"} 2026-07-02T04:08:46.097Z INFO Successfully acquired lease {"lock": "pxc-operator/08db1feb.percona.com"} 2026-07-02T04:08:46.098Z INFO Starting EventSource {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "source": "kind source: *v1.PerconaXtraDBClusterBackup"} 2026-07-02T04:08:46.098Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.PerconaXtraDBCluster"} 2026-07-02T04:08:46.098Z INFO Starting EventSource {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "source": "kind source: *v1.Secret"} 2026-07-02T04:08:46.098Z INFO Starting EventSource {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "source": "kind source: *v1.PerconaXtraDBClusterRestore"} 2026-07-02T04:08:46.198Z INFO Starting Controller {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup"} 2026-07-02T04:08:46.198Z INFO Starting workers {"controller": "pxcbackup-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterBackup", "worker count": 1} 2026-07-02T04:08:46.299Z INFO Starting Controller {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster"} 2026-07-02T04:08:46.299Z INFO Starting Controller {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore"} 2026-07-02T04:08:46.299Z INFO Starting workers {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "worker count": 1} 2026-07-02T04:08:46.299Z INFO Starting workers {"controller": "pxcrestore-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBClusterRestore", "worker count": 1} 2026-07-02T04:09:38.651Z INFO Set CR version {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "version": "1.21.0"} 2026-07-02T04:09:39.111Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:09:39.158Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-proxysql", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}"} 2026-07-02T04:09:39.225Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:09:39.270Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-pxc-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:09:39.314Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-proxysql", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:09:39.462Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a5bb71ea-6fd8-461d-bc66-0e436f8299ae", "object": "scaling-proxysql-unready", "kind": "&TypeMeta{Kind:Service,APIVersion:v1,}"} 2026-07-02T04:09:40.190Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "8eaf2558-e781-4c07-be41-6b3a25d8fdd2", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:09:40.211Z DEBUG Creating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "8eaf2558-e781-4c07-be41-6b3a25d8fdd2", "object": "scaling-proxysql", "kind": "&TypeMeta{Kind:PodDisruptionBudget,APIVersion:policy/v1,}"} 2026-07-02T04:11:06.462Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac", "user": "operator"} 2026-07-02T04:11:06.508Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac", "user": "monitor"} 2026-07-02T04:11:06.592Z INFO User monitor: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac"} 2026-07-02T04:11:06.643Z INFO monitor user privileges granted {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac"} 2026-07-02T04:11:06.697Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac", "user": "xtrabackup"} 2026-07-02T04:11:06.757Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac"} 2026-07-02T04:11:06.845Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac", "user": "replication"} 2026-07-02T04:11:06.861Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0f76be60-9539-49f2-aad7-1eeb6531ccac", "err": "get primary pxc pod: not found"} 2026-07-02T04:11:12.520Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "3a5272bf-3cdb-46ed-9c44-31c47d9e57e3", "err": "get primary pxc pod: not found"} 2026-07-02T04:11:17.638Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "1068d6ab-d829-4304-92b9-ac8e411945b0", "err": "get primary pxc pod: not found"} 2026-07-02T04:13:44.871Z INFO Password expiration policy updated {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "863dc906-8303-4904-ae4e-9742c956b3c8", "user": "root"} 2026-07-02T04:13:45.001Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "863dc906-8303-4904-ae4e-9742c956b3c8", "new version": "8.0.43-34.1"} 2026-07-02T04:13:46.521Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "863dc906-8303-4904-ae4e-9742c956b3c8"} 2026-07-02T04:13:52.674Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "89b145a6-e247-4791-a095-5529aace3cc3"} 2026-07-02T04:13:57.800Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "c3c9834a-d5f7-40c2-9ad6-c59fc9f3999d"} 2026-07-02T04:14:03.099Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "e4604afe-317a-4025-a5d2-bf7da12b059e"} 2026-07-02T04:14:08.782Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "b632c20a-a403-4d81-ba0d-bcbcf86fcb1c"} 2026-07-02T04:14:13.910Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "50fbf7bd-db16-4195-adfa-036099d74969"} 2026-07-02T04:14:19.268Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "89ff724a-155c-4eee-be75-dea57663fb8e"} 2026-07-02T04:14:24.800Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "5216e153-ef31-4b6b-a002-4c2c2cae2a7d"} 2026-07-02T04:14:30.134Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "4b7cb884-b0ae-4910-972e-2e16ef71ebf4"} 2026-07-02T04:14:35.344Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "2852c783-a01f-4c84-b0a0-bbccc9390839"} 2026-07-02T04:14:40.748Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ade4369b-e085-470b-a520-c4df47b3e3ec"} 2026-07-02T04:14:46.041Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "11e74074-6fe8-4568-a35b-b108349ab38f"} 2026-07-02T04:14:51.458Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "b5bd6dde-e03b-4fe3-91d8-87cb085aab32"} 2026-07-02T04:14:57.349Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "66168e68-fd81-47be-94e1-5e239e65a19b"} 2026-07-02T04:15:02.873Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "c80da1a0-ddd6-49e6-95a4-1828c8cdc029"} 2026-07-02T04:15:08.757Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "5d6c4474-cb5c-4058-b11b-aaa004128b42"} 2026-07-02T04:15:13.779Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "163f60d0-b461-46d8-82d2-b2169f3fe9c8"} 2026-07-02T04:15:19.057Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "c36dbd41-ced8-408f-b4fa-f5880f41897e"} 2026-07-02T04:15:24.878Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "aaabf368-4690-4c0d-b88c-72d448b10841"} 2026-07-02T04:15:30.354Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "378b07eb-7556-45ed-95a9-210f541c599c"} 2026-07-02T04:15:35.612Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "1c732ef5-8fa4-4229-b4e4-c165ee6700f5"} 2026-07-02T04:15:40.885Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "69c1509f-4e1a-4ac2-81e9-1cb6c0972843"} 2026-07-02T04:15:46.284Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "365e1f15-0345-4f39-b44a-7496f27b4f65"} 2026-07-02T04:15:51.555Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "dd288b1b-3fbb-4903-a705-6b4919d04615"} 2026-07-02T04:15:56.877Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "933b2650-0311-4b10-bb74-e364a1ae790d"} 2026-07-02T04:16:02.011Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "6bfa5462-bf1a-4ce7-800b-66abe492ef3e"} 2026-07-02T04:16:07.575Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ee3f3b17-dd2a-4098-8719-12855697aea4"} 2026-07-02T04:16:12.796Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "c28ff715-244c-41f1-a919-dd603dce26e2"} 2026-07-02T04:16:15.108Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "b5648bb8-19f4-4ea6-b1fa-0a1f31cb01cc", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:16:15.170Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "b5648bb8-19f4-4ea6-b1fa-0a1f31cb01cc", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:16:16.440Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "b5648bb8-19f4-4ea6-b1fa-0a1f31cb01cc"} 2026-07-02T04:19:09.982Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "76395cde-c078-4589-a721-4786df0b2355"} 2026-07-02T04:19:15.424Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "0e613596-f7c2-40d2-b323-c6b1b27cf1f7"} 2026-07-02T04:19:21.061Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "e15aa807-7493-4d37-9e5f-3141086a4acd"} 2026-07-02T04:19:26.461Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "020b9a85-1299-4c16-b03b-cc8a20b5bb2c"} 2026-07-02T04:19:31.977Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "2e0eec23-ab57-402d-8db5-c985d5b3270f"} 2026-07-02T04:19:37.630Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "f3fc2137-a660-415d-8265-5af3a92acc5c"} 2026-07-02T04:19:42.744Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ef49fdc3-eb79-4b47-ab41-6413ed37da04"} 2026-07-02T04:19:48.256Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "85f141bf-bf23-4899-949a-df4477d54578"} 2026-07-02T04:19:53.921Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "57cca983-6a38-4c55-b693-1fed6f823840"} 2026-07-02T04:19:59.185Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "49d442c3-20ea-4e32-8c9c-da3f9b3b5296"} 2026-07-02T04:20:04.824Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "279cb807-b1fc-4a32-a983-8f34ffe36a41"} 2026-07-02T04:20:10.261Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "8568c64c-cbde-4c02-bb39-a32a53b607a6"} 2026-07-02T04:20:15.665Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "9f4e54ca-aff3-40a8-8075-3329088e4a48"} 2026-07-02T04:20:20.893Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ca775e15-13b3-4470-b8b9-e8b124b2c4ac"} 2026-07-02T04:20:21.777Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "24e93765-4f52-4dc2-9b4a-fc2e602f0fe9", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:20:22.131Z DEBUG Updating object {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "24e93765-4f52-4dc2-9b4a-fc2e602f0fe9", "object": "scaling-pxc", "kind": "&TypeMeta{Kind:StatefulSet,APIVersion:apps/v1,}", "hashChanged": true, "metaChanged": true} 2026-07-02T04:20:23.786Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "24e93765-4f52-4dc2-9b4a-fc2e602f0fe9"} 2026-07-02T04:20:34.211Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "fc50d828-8447-46a7-b8ed-3379ed47aa33", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-4: dial tcp 10.26.74.4:33062: connect: connection refused"} 2026-07-02T04:20:44.828Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "fc78cb81-3652-4391-8be5-feee181d6d41", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-3: dial tcp: lookup scaling-pxc-3.scaling-pxc.scaling-28830 on 34.118.224.10:53: no such host"} 2026-07-02T04:20:50.584Z INFO reconcile replication error {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "a92e3467-4cb9-4885-baa8-5e3fb715a150", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-3: dial tcp: lookup scaling-pxc-3.scaling-pxc.scaling-28830 on 34.118.224.10:53: no such host"} 2026-07-02T04:20:57.995Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "80c19871-4ea3-4e5d-b5f7-949bed37d298"} 2026-07-02T04:21:03.428Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ef171bc6-e898-47cc-822d-a83887464400"} 2026-07-02T04:21:08.864Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "ecdc6f2c-8d66-4502-a51a-58d0d4ec558f"} 2026-07-02T04:21:14.223Z DEBUG PXC users synced with ProxySQL {"controller": "pxc-controller", "controllerGroup": "pxc.percona.com", "controllerKind": "PerconaXtraDBCluster", "PerconaXtraDBCluster": {"name":"scaling","namespace":"scaling-28830"}, "namespace": "scaling-28830", "name": "scaling", "reconcileID": "d3e7b3be-4a29-4609-af07-0a0d7457baf8"} -  }, -  { -  }, +  }, -  Annotations: map[string]string{ +  Annotations: map[string]string{ -  APIVersion: "apps/v1", -  APIVersion: "apps/v1", +  AvailableReplicas: 0, -  AvailableReplicas: 3, -  AvailableReplicas: 5, -  CollisionCount: &0, +  CollisionCount: nil, +  CreationTimestamp: v1.Time{}, -  CreationTimestamp: v1.Time{Time: s"2026-07-02 04:09:39 +0000 UTC"}, +  CurrentReplicas: 0, -  CurrentReplicas: 3, -  CurrentReplicas: 5, +  CurrentRevision: "", -  CurrentRevision: "scaling-pxc-586444557b", -  DefaultMode: &420, -  DefaultMode: &420, +  DefaultMode: nil, +  DefaultMode: nil, +  DeprecatedServiceAccount: "", -  DeprecatedServiceAccount: "default", +  DNSPolicy: "", -  DNSPolicy: "ClusterFirst", -  FieldsType: "FieldsV1", -  FieldsType: "FieldsV1", -  FieldsV1: s`{"f:metadata":{"f:annotations":{".":{},"f:percona.com/last-confi`..., -  FieldsV1: s`{"f:status":{"f:availableReplicas":{},"f:collisionCount":{},"f:c`..., +  Generation: 0, -  Generation: 1, -  Generation: 2, +  ManagedFields: nil, -  ManagedFields: []v1.ManagedFieldsEntry{ -  Manager: "kube-controller-manager", -  Manager: "percona-xtradb-cluster-operator", +  ObservedGeneration: 0, -  ObservedGeneration: 1, -  ObservedGeneration: 2, -  Operation: "Update", -  Operation: "Update", -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzY2FsaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJu"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6Mywic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzY2FsaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJu"..., -  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NSwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzY2FsaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJu"..., +  "percona.com/last-config-hash": "eyJyZXBsaWNhcyI6NSwic2VsZWN0b3IiOnsibWF0Y2hMYWJlbHMiOnsiYXBwLmt1YmVybmV0ZXMuaW8vY29tcG9uZW50IjoicHhjIiwiYXBwLmt1YmVybmV0ZXMuaW8vaW5zdGFuY2UiOiJzY2FsaW5nIiwiYXBwLmt1YmVybmV0ZXMuaW8vbWFuYWdlZC1ieSI6InBlcmNvbmEteHRyYWRiLWNsdXN0ZXItb3BlcmF0b3IiLCJhcHAua3ViZXJu"..., +  PeriodSeconds: 0, -  PeriodSeconds: 10, +  PersistentVolumeClaimRetentionPolicy: nil, -  PersistentVolumeClaimRetentionPolicy: s"&StatefulSetPersistentVolumeClaimRetentionPolicy{WhenDeleted:Retain,WhenScaled:Retain,}", +  Phase: "", -  Phase: "Pending", +  PodManagementPolicy: "", -  PodManagementPolicy: "OrderedReady", +  Protocol: "", -  Protocol: "TCP", +  ReadyReplicas: 0, -  ReadyReplicas: 3, -  ReadyReplicas: 5, +  Replicas: 0, -  Replicas: 3, -  Replicas: &3, +  Replicas: &3, -  Replicas: 5, -  Replicas: &5, +  Replicas: &5, +  ResourceVersion: "", -  ResourceVersion: "1782965624071295020", -  ResourceVersion: "1782965942618063020", +  RestartPolicy: "", -  RestartPolicy: "Always", -  RevisionHistoryLimit: &10, +  RevisionHistoryLimit: nil, +  SchedulerName: "", -  SchedulerName: "default-scheduler", -  Subresource: "status", +  TerminationMessagePath: "", -  TerminationMessagePath: "/dev/termination-log", +  TerminationMessagePolicy: "", -  TerminationMessagePolicy: "File", -  Time: s"2026-07-02 04:09:39 +0000 UTC", -  Time: s"2026-07-02 04:13:44 +0000 UTC", -  Time: s"2026-07-02 04:16:15 +0000 UTC", -  Time: s"2026-07-02 04:19:02 +0000 UTC", -  TopologySpreadConstraints: nil, +  TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, +  UID: "", -  UID: "42c09e68-1365-4048-9067-1894b394d96c", +  UpdatedReplicas: 0, -  UpdatedReplicas: 3, -  UpdatedReplicas: 5, +  UpdateRevision: "", -  UpdateRevision: "scaling-pxc-586444557b", -  VolumeMode: &"Filesystem", +  VolumeMode: nil,   }    },    },    {    },    },    {    },    },    {    },    ... // 16 identical fields    ... // 16 identical fields    ... // 22 identical fields    ... // 2 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 3 identical fields    ... // 4 identical fields    ... // 5 identical fields    ... // 6 identical fields    ... // 7 identical fields    ... // 9 identical fields    AccessModes: nil,    ActiveDeadlineSeconds: nil,    Affinity: nil,    Args: {"mysqld"},    AutomountServiceAccountToken: nil,    AWSElasticBlockStore: nil,    AzureFile: nil,    Capacity: nil,    Conditions: nil,    ConfigMap: &v1.ConfigMapVolumeSource{    ContainerPort: 3306,    ContainerPort: 33060,    ContainerPort: 33062,    ContainerPort: 4444,    ContainerPort: 4567,    ContainerPort: 4568,    Containers: []v1.Container{    DataSource: nil,    DataSourceRef: nil,    DeletionGracePeriodSeconds: nil,    DeletionTimestamp: nil,    EnvFrom: {{SecretRef: &{LocalObjectReference: {Name: "scaling-env-vars-pxc"}, Optional: &true}}},    Env: {{Name: "PXC_SERVICE", Value: "scaling-pxc-unready"}, {Name: "MONITOR_HOST", Value: "%"}, {Name: "MYSQL_ROOT_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-scaling"}, Key: "root"}}}, {Name: "XTRABACKUP_PASSWORD", ValueFrom: &{SecretKeyRef: &{LocalObjectReference: {Name: "internal-scaling"}, Key: "xtrabackup"}}}, ...},    EphemeralContainers: nil,    FailureThreshold: 3,    FC: nil,    Finalizers: nil,    GitRepo: nil,    HostAliases: nil,    HostIP: "",    HostPort: 0,    ImagePullPolicy: "Always",    InitContainers: []v1.Container{    InitialDelaySeconds: 300,    ISCSI: nil,    Items: nil,    Items: nil,    Labels: nil,    Lifecycle: nil,    LivenessProbe: &v1.Probe{    LocalObjectReference: {Name: "auto-scaling-pxc"},    LocalObjectReference: {Name: "scaling-pxc"},    MinReadySeconds: 0,    Name: "auto-config",    {Name: "bin", VolumeSource: {EmptyDir: &{}}},    Name: "config",    Name: "ist",    Name: "mysql",    Name: "mysql-admin",    Name: "mysql-init-file",    Name: "mysql-users-secret-file",    Name: "mysqlx",    Namespace: "scaling-28830",    Name: "ssl",    Name: "ssl-internal",    Name: "sst",    {Name: "tmp", VolumeSource: {EmptyDir: &{}}},    Name: "vault-keyring-secret",    Name: "write-set",    NFS: nil,    NodeName: "",    NodeSelector: nil,    ObjectMeta: {Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "scaling", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}, Annotations: {"kubectl.kubernetes.io/default-container": "pxc", "percona.com/configuration-hash": "d41d8cd98f00b204e9800998ecf8427e", "percona.com/ssl-hash": "cfd6a52398268173b51d7cdb1331c09a", "percona.com/ssl-internal-hash": "751e59a1c1d94ac67f9d6bb6e756acdc"}},    ObjectMeta: {Name: "datadir", Labels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "scaling", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    ObjectMeta: v1.ObjectMeta{    Optional: &false,    Optional: &true,    Optional: &true,    Ordinals: nil,    OS: nil,    Overhead: nil,    OwnerReferences: {{APIVersion: "pxc.percona.com/v1", Kind: "PerconaXtraDBCluster", Name: "scaling", UID: "4a8bec5b-f17f-43e4-8129-51d96a6f7920", ...}},    Ports: []v1.ContainerPort{    PreemptionPolicy: nil,    ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/liveness-check.sh"}}},    Quobyte: nil,    ReadinessProbe: &{ProbeHandler: {Exec: &{Command: {"/var/lib/mysql/readiness-check.sh"}}}, InitialDelaySeconds: 15, TimeoutSeconds: 15, PeriodSeconds: 30, ...},    SecretName: "internal-scaling",    SecretName: "scaling-mysql-init",    SecretName: "scaling-vault",    SecretName: "some-name-ssl",    SecretName: "some-name-ssl-internal",    Secret: &v1.SecretVolumeSource{    SecurityContext: nil,    Selector: &{MatchLabels: {"app.kubernetes.io/component": "pxc", "app.kubernetes.io/instance": "scaling", "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", "app.kubernetes.io/name": "percona-xtradb-cluster", ...}},    SelfLink: "",    ServiceAccountName: "default",    ServiceName: "scaling-pxc",    SetHostnameAsFQDN: nil,    Spec: v1.PersistentVolumeClaimSpec{    Spec: v1.PodSpec{    Spec: v1.StatefulSetSpec{    StartupProbe: nil,    Status: v1.PersistentVolumeClaimStatus{    Status: v1.StatefulSetStatus{    StorageClassName: nil,    Subdomain: "",    SuccessThreshold: 1,    Template: v1.PodTemplateSpec{    TerminationGracePeriodSeconds: &600,    TerminationGracePeriodSeconds: nil,    TimeoutSeconds: 5,    Tolerations: nil,    TypeMeta: {},    TypeMeta: {Kind: "StatefulSet", APIVersion: "apps/v1"},    UpdateStrategy: {Type: "RollingUpdate", RollingUpdate: &{Partition: &0}},   &v1.StatefulSet{    VolumeAttributesClassName: nil,    VolumeClaimTemplates: []v1.PersistentVolumeClaim{    VolumeDevices: nil,    VolumeMounts: {{Name: "datadir", MountPath: "/var/lib/mysql"}, {Name: "config", MountPath: "/etc/percona-xtradb-cluster.conf.d"}, {Name: "tmp", MountPath: "/tmp"}, {Name: "ssl", MountPath: "/etc/mysql/ssl"}, ...},    VolumeName: "",    VolumeSource: v1.VolumeSource{    Volumes: []v1.Volume{    VsphereVolume: nil,    WorkingDir: "", + local backup_names ++ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + backup_names= + [[ -n '' ]] + local pxc_names ++ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}' + pxc_names=scaling-28830/scaling + [[ -n scaling-28830/scaling ]] + read -r pns name + kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}' + kubectl_bin patch pxc -n scaling-28830 scaling --type=merge -p '{"metadata":{"finalizers":["percona.com/delete-pxc-pvc"]}}' perconaxtradbcluster.pxc.percona.com/scaling patched + read -r pns name + timeout 300 kubectl delete pxc --all --all-namespaces --ignore-not-found perconaxtradbcluster.pxc.percona.com "scaling" deleted from scaling-28830 namespace ++ date +%s + local deadline=1782966389 ++ kubectl_bin get pxc --all-namespaces -o name --ignore-not-found + [[ -z '' ]] + timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found No resources found + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found ++ mktemp + local LAST_OUT=/tmp/tmp.p4TaPN92e3 ++ mktemp + local LAST_ERR=/tmp/tmp.veyin0vpmV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook --ignore-not-found + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.p4TaPN92e3 validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.veyin0vpmV + rm /tmp/tmp.p4TaPN92e3 /tmp/tmp.veyin0vpmV + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' ++ date +%s + local deadline=1782966241 + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace scaling-28830 + local LAST_OUT=/tmp/tmp.P0qnLsorPx ++ mktemp + local LAST_ERR=/tmp/tmp.JSr10rsSqT + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.E4hr6VNFIf ++ mktemp + local LAST_ERR=/tmp/tmp.OiTzluRIMZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace scaling-28830 + [[ -z namespace/scaling-28830 ]] ++ date +%s + [[ 1782966124 -ge 1782966241 ]] + sleep 5 ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name + [[ -z namespace/scaling-28830 ]] ++ date +%s + [[ 1782966131 -ge 1782966241 ]] + sleep 5 ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name + [[ -z namespace/scaling-28830 ]] ++ date +%s + [[ 1782966139 -ge 1782966241 ]] + sleep 5 ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name + [[ -z namespace/scaling-28830 ]] ++ date +%s + [[ 1782966147 -ge 1782966241 ]] + sleep 5 ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name + [[ -z namespace/scaling-28830 ]] ++ date +%s + [[ 1782966155 -ge 1782966241 ]] + sleep 5 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.E4hr6VNFIf namespace "scaling-28830" force deleted + cat /tmp/tmp.OiTzluRIMZ Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. + rm /tmp/tmp.E4hr6VNFIf /tmp/tmp.OiTzluRIMZ + return 0 ++ kubectl_bin get namespace scaling-28830 --ignore-not-found -o name + [[ -z '' ]] ++ date +%s + deadline=1782966283 ++ kubectl_bin get pv -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' ++ awk '$2 != ""' + [[ -z '' ]] + rm -rf /tmp/tmp.MhYBIbMudc + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed -----------------------------------------------------------------------------------