Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/logs/scaling-8-0.log
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
+ create_infra scaling-6084
+ local ns=scaling-6084
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n scaling-32088 scaling --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/scaling patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.91PGjRRaC7
++ mktemp
+ local LAST_ERR=/tmp/tmp.40McyGum5T
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.91PGjRRaC7
perconaxtradbcluster.pxc.percona.com "scaling" deleted
+ cat /tmp/tmp.40McyGum5T
+ rm /tmp/tmp.91PGjRRaC7 /tmp/tmp.40McyGum5T
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ei04zwXuCQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.DM8eblh3Na
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ei04zwXuCQ
No resources found
+ cat /tmp/tmp.DM8eblh3Na
+ rm /tmp/tmp.Ei04zwXuCQ /tmp/tmp.DM8eblh3Na
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.PWg8LNsXFS
++ mktemp
+ local LAST_ERR=/tmp/tmp.bRMKWcknkG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.PWg8LNsXFS
No resources found
+ cat /tmp/tmp.bRMKWcknkG
+ rm /tmp/tmp.PWg8LNsXFS /tmp/tmp.bRMKWcknkG
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.jGLfFsoIaw
+ local LAST_OUT=/tmp/tmp.zERjn8NnFO
+ xargs kubectl delete ns
++ mktemp
++ mktemp
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ local LAST_ERR=/tmp/tmp.KrFLBI4rko
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ local LAST_ERR=/tmp/tmp.0068QsGyUk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jGLfFsoIaw
+ cat /tmp/tmp.0068QsGyUk
+ rm /tmp/tmp.jGLfFsoIaw /tmp/tmp.0068QsGyUk
+ return 0
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "scaling-32088" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zERjn8NnFO
namespace "pxc-operator" deleted
+ cat /tmp/tmp.KrFLBI4rko
+ rm /tmp/tmp.zERjn8NnFO /tmp/tmp.KrFLBI4rko
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.3zDOFb9xL3
++ mktemp
+ local LAST_ERR=/tmp/tmp.xU9RO6m2Wk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3zDOFb9xL3
namespace/pxc-operator created
+ cat /tmp/tmp.xU9RO6m2Wk
+ rm /tmp/tmp.3zDOFb9xL3 /tmp/tmp.xU9RO6m2Wk
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cFMl6d6LYh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.qHOnMEBvxF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cFMl6d6LYh
++ cat /tmp/tmp.qHOnMEBvxF
++ rm /tmp/tmp.cFMl6d6LYh /tmp/tmp.qHOnMEBvxF
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YetfF2PR92
++ mktemp
+ local LAST_ERR=/tmp/tmp.YZBHnit7pz
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YetfF2PR92
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6" modified.
+ cat /tmp/tmp.YZBHnit7pz
+ rm /tmp/tmp.YetfF2PR92 /tmp/tmp.YZBHnit7pz
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.J6n6gfCN5v
++ mktemp
+ local LAST_ERR=/tmp/tmp.ScHOHCDirV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.J6n6gfCN5v
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.ScHOHCDirV
+ rm /tmp/tmp.J6n6gfCN5v /tmp/tmp.ScHOHCDirV
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ye2he9qe3J
++ mktemp
+ local LAST_ERR=/tmp/tmp.mFiAKl7OUd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ye2he9qe3J
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.mFiAKl7OUd
+ rm /tmp/tmp.ye2he9qe3J /tmp/tmp.mFiAKl7OUd
+ return 0
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.rGQF41QjDd
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
++ mktemp
+ local LAST_ERR=/tmp/tmp.TDIpeoSJLF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rGQF41QjDd
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.TDIpeoSJLF
+ rm /tmp/tmp.rGQF41QjDd /tmp/tmp.TDIpeoSJLF
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.1AcNLZFA9k
++ mktemp
+ local LAST_ERR=/tmp/tmp.c8vydJbMnF
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1AcNLZFA9k
pod/percona-xtradb-cluster-operator-67cbddd7f9-5rwc4 condition met
+ cat /tmp/tmp.c8vydJbMnF
+ rm /tmp/tmp.1AcNLZFA9k /tmp/tmp.c8vydJbMnF
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MbHb8OZzSA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nxkZTJxSa7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.MbHb8OZzSA
++ cat /tmp/tmp.nxkZTJxSa7
++ rm /tmp/tmp.MbHb8OZzSA /tmp/tmp.nxkZTJxSa7
++ return 0
+ wait_pod percona-xtradb-cluster-operator-67cbddd7f9-5rwc4 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-67cbddd7f9-5rwc4
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-67cbddd7f9-5rwc4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-67cbddd7f9-5rwc4 condition met
waiting for pod/percona-xtradb-cluster-operator-67cbddd7f9-5rwc4 to become Ready.Ok
+ sleep 3
+ create_namespace scaling-6084
+ local namespace=scaling-6084
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces scaling-6084'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces scaling-6084
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace scaling-6084
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.DwCBQbqoXV
++ mktemp
+ local LAST_ERR=/tmp/tmp.G4AnebgPGc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scaling-6084
++ mktemp
+ local LAST_OUT=/tmp/tmp.nTTaXkcYOY
++ mktemp
+ local LAST_ERR=/tmp/tmp.VA4PulbdHe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scaling-6084
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nTTaXkcYOY
+ cat /tmp/tmp.VA4PulbdHe
+ rm /tmp/tmp.nTTaXkcYOY /tmp/tmp.VA4PulbdHe
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace scaling-6084
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.DwCBQbqoXV
+ cat /tmp/tmp.G4AnebgPGc
Error from server (NotFound): namespaces "scaling-6084" not found
+ rm /tmp/tmp.DwCBQbqoXV /tmp/tmp.G4AnebgPGc
+ return 1
+ :
+ wait_for_delete namespace/scaling-6084
+ local res=namespace/scaling-6084
+ echo -n 'waiting for namespace/scaling-6084 to be deleted'
waiting for namespace/scaling-6084 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "scaling-6084" not found
+ desc 'create namespace scaling-6084'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace scaling-6084
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace scaling-6084
++ mktemp
+ local LAST_OUT=/tmp/tmp.6VQitcsVPz
++ mktemp
+ local LAST_ERR=/tmp/tmp.EniNX8VSRl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace scaling-6084
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6VQitcsVPz
namespace/scaling-6084 created
+ cat /tmp/tmp.EniNX8VSRl
+ rm /tmp/tmp.6VQitcsVPz /tmp/tmp.EniNX8VSRl
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.R3DXqsJhRx
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b5qrbV1Dtr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.R3DXqsJhRx
++ cat /tmp/tmp.b5qrbV1Dtr
++ rm /tmp/tmp.R3DXqsJhRx /tmp/tmp.b5qrbV1Dtr
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6 --namespace=scaling-6084
++ mktemp
+ local LAST_OUT=/tmp/tmp.829L8n6Mnf
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZwJwyOTMyu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6 --namespace=scaling-6084
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.829L8n6Mnf
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster6" modified.
+ cat /tmp/tmp.ZwJwyOTMyu
+ rm /tmp/tmp.829L8n6Mnf /tmp/tmp.ZwJwyOTMyu
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.5RE39ta9tF
++ mktemp
+ local LAST_ERR=/tmp/tmp.WPDihMF2nS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5RE39ta9tF
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.WPDihMF2nS
+ rm /tmp/tmp.5RE39ta9tF /tmp/tmp.WPDihMF2nS
+ return 0
+ spinup_pxc scaling /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ local cluster=scaling
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.R4jXl9qnsL
++ mktemp
+ local LAST_ERR=/tmp/tmp.hnXne1Gk8u
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.R4jXl9qnsL
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.hnXne1Gk8u
+ rm /tmp/tmp.R4jXl9qnsL /tmp/tmp.hnXne1Gk8u
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-6084~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.HbxI2a3PWq
++ mktemp
+ local LAST_ERR=/tmp/tmp.tg6AiKjiTp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HbxI2a3PWq
deployment.apps/pxc-client created
+ cat /tmp/tmp.tg6AiKjiTp
+ rm /tmp/tmp.HbxI2a3PWq /tmp/tmp.tg6AiKjiTp
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-6084~
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_OUT=/tmp/tmp.Hx5mHjqCmI
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.WW8gaglGrt
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Hx5mHjqCmI
perconaxtradbcluster.pxc.percona.com/scaling created
+ cat /tmp/tmp.WW8gaglGrt
+ rm /tmp/tmp.Hx5mHjqCmI /tmp/tmp.WW8gaglGrt
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy scaling
++ local target_cluster=scaling
+++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.PgEiEo2Avq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.roAdaOXxVN
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc scaling -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.PgEiEo2Avq
+++ cat /tmp/tmp.roAdaOXxVN
+++ rm /tmp/tmp.PgEiEo2Avq /tmp/tmp.roAdaOXxVN
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2ZS93rdEhq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oTIS5F9sep
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc scaling -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.2ZS93rdEhq
+++ cat /tmp/tmp.oTIS5F9sep
+++ rm /tmp/tmp.2ZS93rdEhq /tmp/tmp.oTIS5F9sep
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo scaling-proxysql
++ return
+ local proxy=scaling-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scaling-6084
++ mktemp
+ local LAST_OUT=/tmp/tmp.obTt5LZUFP
++ mktemp
+ local LAST_ERR=/tmp/tmp.lWeSxPHIh2
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scaling-6084
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scaling-6084
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n scaling-6084
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.obTt5LZUFP
+ cat /tmp/tmp.lWeSxPHIh2
error: no matching resources found
+ rm /tmp/tmp.obTt5LZUFP /tmp/tmp.lWeSxPHIh2
+ return 1
+ true
+ wait_for_running scaling-proxysql 1
+ local name=scaling-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-proxysql-0 480
+ local pod=scaling-proxysql-0
+ local max_retry=480
+ local ns=
++ echo scaling-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/scaling-proxysql-0 condition met
waiting for pod/scaling-proxysql-0 to become Ready.Ok
+ wait_for_running scaling-pxc 3
+ local name=scaling-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-0 480
+ local pod=scaling-pxc-0
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-0 condition met
waiting for pod/scaling-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-1 480
+ local pod=scaling-pxc-1
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-1 condition met
waiting for pod/scaling-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-2 480
+ local pod=scaling-pxc-2
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-2 condition met
waiting for pod/scaling-pxc-2 to become Ready.Ok
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h scaling-proxysql -uroot -proot_password -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h scaling-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9vnAXgdLJv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M3qkrS0hCA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9vnAXgdLJv
++ cat /tmp/tmp.M3qkrS0hCA
++ rm /tmp/tmp.9vnAXgdLJv /tmp/tmp.M3qkrS0hCA
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h scaling-proxysql -uroot -proot_password -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h scaling-proxysql -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.U76mz7i8sy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Q7eh78fgiD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.U76mz7i8sy
++ cat /tmp/tmp.Q7eh78fgiD
++ rm /tmp/tmp.U76mz7i8sy /tmp/tmp.Q7eh78fgiD
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-69f9bd646c-tx2jz
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-0.scaling-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-0.scaling-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-0.scaling-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-0.scaling-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sdWUBnRT0a
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gLr6VlmOzw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sdWUBnRT0a
++ cat /tmp/tmp.gLr6VlmOzw
++ rm /tmp/tmp.sdWUBnRT0a /tmp/tmp.gLr6VlmOzw
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.XRBAG9Aezm/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.XRBAG9Aezm/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-1.scaling-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-1.scaling-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-1.scaling-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-1.scaling-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VlRLLHqG2L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kyssu40irJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VlRLLHqG2L
++ cat /tmp/tmp.kyssu40irJ
++ rm /tmp/tmp.VlRLLHqG2L /tmp/tmp.kyssu40irJ
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.XRBAG9Aezm/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.XRBAG9Aezm/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-2.scaling-pxc -uroot -proot_password -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-2.scaling-pxc -uroot -proot_password -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-2.scaling-pxc -uroot -proot_password -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-2.scaling-pxc -uroot -proot_password -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tjAEzcgqCY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Vsh8foO5Ny
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tjAEzcgqCY
++ cat /tmp/tmp.Vsh8foO5Ny
++ rm /tmp/tmp.tjAEzcgqCY /tmp/tmp.Vsh8foO5Ny
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.XRBAG9Aezm/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.XRBAG9Aezm/select-1.sql
++ is_keyring_plugin_in_use scaling
++ local cluster=scaling
++ kubectl_bin exec -it scaling-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FJ8VuGjUyA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lLRFNzzgij
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it scaling-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FJ8VuGjUyA
++ cat /tmp/tmp.lLRFNzzgij
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.FJ8VuGjUyA /tmp/tmp.lLRFNzzgij
++ return 0
+ '[' '' ']'
+ desc 'scale up from 3 to 5'
+ set +o xtrace


-----------------------------------------------------------------------------------
scale up from 3 to 5
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f-
+ sed -e 's/size: 3/size: 5/'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-6084~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UyLlT87Wzs
++ mktemp
+ local LAST_ERR=/tmp/tmp.99hdjZQqEu
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UyLlT87Wzs
perconaxtradbcluster.pxc.percona.com/scaling configured
+ cat /tmp/tmp.99hdjZQqEu
+ rm /tmp/tmp.UyLlT87Wzs /tmp/tmp.99hdjZQqEu
+ return 0
+ desc 'check if all 5 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 5 Pods started
-----------------------------------------------------------------------------------

+ wait_for_running scaling-pxc 5
+ local name=scaling-pxc
+ let last_pod=4
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 4
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-0 480
+ local pod=scaling-pxc-0
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-0 condition met
waiting for pod/scaling-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-1 480
+ local pod=scaling-pxc-1
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-1
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-1 condition met
waiting for pod/scaling-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-2 480
+ local pod=scaling-pxc-2
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-2 condition met
waiting for pod/scaling-pxc-2 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-3 480
+ local pod=scaling-pxc-3
+ local max_retry=480
+ local ns=
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo scaling-pxc-3
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-3 condition met
waiting for pod/scaling-pxc-3 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod scaling-pxc-4 480
+ local pod=scaling-pxc-4
+ local max_retry=480
+ local ns=
++ echo scaling-pxc-4
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/scaling-pxc-4 condition met
waiting for pod/scaling-pxc-4 to become Ready.Ok
+ sleep 15
+ desc 'check if PVC created'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if PVC created
-----------------------------------------------------------------------------------

+ compare_kubectl pvc/datadir-scaling-pxc-3
+ local resource=pvc/datadir-scaling-pxc-3
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml
+ local new_result=/tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-3.yml
+ desc 'compare pvc/datadir-scaling-pxc-3-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare pvc/datadir-scaling-pxc-3-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k127.yml ']'
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']'
+ kubectl_bin get -o yaml pvc/datadir-scaling-pxc-3
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("scaling-6084", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.fQe539CsMw
++ mktemp
+ local LAST_ERR=/tmp/tmp.PpGcK3GCUs
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml pvc/datadir-scaling-pxc-3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fQe539CsMw
+ cat /tmp/tmp.PpGcK3GCUs
+ rm /tmp/tmp.fQe539CsMw /tmp/tmp.PpGcK3GCUs
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml /tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-3.yml
+ compare_kubectl pvc/datadir-scaling-pxc-4
+ local resource=pvc/datadir-scaling-pxc-4
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml
+ local new_result=/tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-4.yml
+ desc 'compare pvc/datadir-scaling-pxc-4-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare pvc/datadir-scaling-pxc-4-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k127.yml ']'
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']'
+ kubectl_bin get -o yaml pvc/datadir-scaling-pxc-4
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("scaling-6084", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hN6H4zwf2c
++ mktemp
+ local LAST_ERR=/tmp/tmp.AZO8BOKoiX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml pvc/datadir-scaling-pxc-4
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.hN6H4zwf2c
+ cat /tmp/tmp.AZO8BOKoiX
+ rm /tmp/tmp.hN6H4zwf2c /tmp/tmp.AZO8BOKoiX
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml /tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-4.yml
+ desc 'check data consistency for new Pods'
+ set +o xtrace


-----------------------------------------------------------------------------------
check data consistency for new Pods
-----------------------------------------------------------------------------------

+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-3.scaling-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-3.scaling-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-3.scaling-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-3.scaling-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zp8KudGliq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2E7I3tst7l
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Zp8KudGliq
++ cat /tmp/tmp.2E7I3tst7l
++ rm /tmp/tmp.Zp8KudGliq /tmp/tmp.2E7I3tst7l
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.XRBAG9Aezm/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.XRBAG9Aezm/select-1.sql
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h scaling-pxc-4.scaling-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-4.scaling-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h scaling-pxc-4.scaling-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h scaling-pxc-4.scaling-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vchliarRN1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GxjV6TbTyE
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vchliarRN1
++ cat /tmp/tmp.GxjV6TbTyE
++ rm /tmp/tmp.vchliarRN1 /tmp/tmp.GxjV6TbTyE
++ return 0
+ client_pod=pxc-client-69f9bd646c-tx2jz
+ wait_pod pxc-client-69f9bd646c-tx2jz
+ local pod=pxc-client-69f9bd646c-tx2jz
+ local max_retry=480
+ local ns=
++ echo pxc-client-69f9bd646c-tx2jz
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-69f9bd646c-tx2jz condition met
waiting for pod/pxc-client-69f9bd646c-tx2jz to become Ready.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.XRBAG9Aezm/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/select-1.sql /tmp/tmp.XRBAG9Aezm/select-1.sql
+ sleep 15
+ desc 'check new Pods exists in ProxySQL'
+ set +o xtrace


-----------------------------------------------------------------------------------
check new Pods exists in ProxySQL
-----------------------------------------------------------------------------------

+ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0
+ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+ local pod=scaling-proxysql-0
+ local container_name=
+ set +o xtrace
+ grep scaling-pxc-3
scaling-pxc-3.scaling-pxc.scaling-6084.svc.cluster.local
scaling-pxc-3.scaling-pxc.scaling-6084.svc.cluster.local
+ grep scaling-pxc-4
+ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0
+ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'';'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+ local pod=scaling-proxysql-0
+ local container_name=
+ set +o xtrace
scaling-pxc-4.scaling-pxc.scaling-6084.svc.cluster.local
scaling-pxc-4.scaling-pxc.scaling-6084.svc.cluster.local
+ desc 'scale down from 5 to 3'
+ set +o xtrace


-----------------------------------------------------------------------------------
scale down from 5 to 3
-----------------------------------------------------------------------------------

+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.scaling-6084~
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.SUZfpSf3uv
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/conf/scaling.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.K6RjzkIT7s
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SUZfpSf3uv
perconaxtradbcluster.pxc.percona.com/scaling configured
+ cat /tmp/tmp.K6RjzkIT7s
+ rm /tmp/tmp.SUZfpSf3uv /tmp/tmp.K6RjzkIT7s
+ return 0
+ sleep 25
+ desc 'check if Pod deleted'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod deleted
-----------------------------------------------------------------------------------

+ wait_for_delete pod/scaling-pxc-3
+ local res=pod/scaling-pxc-3
+ echo -n 'waiting for pod/scaling-pxc-3 to be deleted'
waiting for pod/scaling-pxc-3 to be deleted+ set +o xtrace
..Error from server (NotFound): pods "scaling-pxc-3" not found
+ wait_for_delete pod/scaling-pxc-4
+ local res=pod/scaling-pxc-4
+ echo -n 'waiting for pod/scaling-pxc-4 to be deleted'
waiting for pod/scaling-pxc-4 to be deleted+ set +o xtrace
Error from server (NotFound): pods "scaling-pxc-4" not found
+ desc 'check if PVC not deleted'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if PVC not deleted
-----------------------------------------------------------------------------------

+ compare_kubectl pvc/datadir-scaling-pxc-3
+ local resource=pvc/datadir-scaling-pxc-3
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml
+ local new_result=/tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-3.yml
+ desc 'compare pvc/datadir-scaling-pxc-3-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare pvc/datadir-scaling-pxc-3-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.28 >= 1.29'
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.28 >= 1.27'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k127.yml ']'
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.28 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.28 >= 1.21'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.28 >= 1.29'
+ '[' 0 -eq 1 ']'
+ return 1
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3-eks.yml ']'
+ kubectl_bin get -o yaml pvc/datadir-scaling-pxc-3
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("scaling-6084", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.Uw4t1ELc2B
++ mktemp
+ local LAST_ERR=/tmp/tmp.VR6Qik7zKO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml pvc/datadir-scaling-pxc-3
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Uw4t1ELc2B
+ cat /tmp/tmp.VR6Qik7zKO
+ rm /tmp/tmp.Uw4t1ELc2B /tmp/tmp.VR6Qik7zKO
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-3.yml /tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-3.yml
+ compare_kubectl pvc/datadir-scaling-pxc-4
+ local resource=pvc/datadir-scaling-pxc-4
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml
+ local new_result=/tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-4.yml
+ desc 'compare pvc/datadir-scaling-pxc-4-'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare pvc/datadir-scaling-pxc-4-
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-80.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.27'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k127.yml ']'
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-oc.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.28 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4-eks.yml ']'
+ kubectl_bin get -o yaml pvc/datadir-scaling-pxc-4
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.labels."batch.kubernetes.io/job-name") |
			del(.metadata.labels."job-name") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.spec.template.metadata.labels."batch.kubernetes.io/job-name") |
			del(.spec.template.metadata.labels."job-name") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("scaling-6084", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.OPMTqYaIqE
++ mktemp
+ local LAST_ERR=/tmp/tmp.KlfPwXJyzB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml pvc/datadir-scaling-pxc-4
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.OPMTqYaIqE
+ cat /tmp/tmp.KlfPwXJyzB
+ rm /tmp/tmp.OPMTqYaIqE /tmp/tmp.KlfPwXJyzB
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/scaling/compare/pvc_datadir-scaling-pxc-4.yml /tmp/tmp.XRBAG9Aezm/pvc_datadir-scaling-pxc-4.yml
+ desc 'check if Pod deleted from ProxySQL'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if Pod deleted from ProxySQL
-----------------------------------------------------------------------------------

+ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0
+ grep scaling-pxc-3
+ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+ local pod=scaling-proxysql-0
+ local container_name=
+ set +o xtrace
+ :
+ grep scaling-pxc-4
+ run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;' '-h127.0.0.1 -P6032 -uproxyadmin -padmin_password' scaling-proxysql-0
+ local 'command=SELECT hostname FROM runtime_mysql_servers WHERE status='\''ONLINE'\'' AND hostgroup_id!=13;'
+ local 'uri=-h127.0.0.1 -P6032 -uproxyadmin -padmin_password'
+ local pod=scaling-proxysql-0
+ local container_name=
+ set +o xtrace
+ :
+ destroy scaling-6084
+ local namespace=scaling-6084
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
+ grep -v 'the object has been modified'
+ grep -v 'get backup status: Job.batch'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
+ tee /tmp/tmp.XRBAG9Aezm/operator.log
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GLOasfyUXl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FUe4WT368K
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GLOasfyUXl
++ cat /tmp/tmp.FUe4WT368K
++ rm /tmp/tmp.GLOasfyUXl /tmp/tmp.FUe4WT368K
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-67cbddd7f9-5rwc4
++ mktemp
+ local LAST_OUT=/tmp/tmp.lZtGMmfrEN
++ mktemp
+ local LAST_ERR=/tmp/tmp.Imgbz3UyAw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-67cbddd7f9-5rwc4
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lZtGMmfrEN
+ cat /tmp/tmp.Imgbz3UyAw
+ rm /tmp/tmp.lZtGMmfrEN /tmp/tmp.Imgbz3UyAw
+ return 0
2024-12-20T14:35:33.284Z	INFO	setup	Manager starting up	{"gitCommit": "26afa5272972c583e9dc3f39e65e29a2744c72d8", "gitBranch": "PR-1697-26afa527", "buildTime": "2024-12-20T13:32:57Z", "goVersion": "go1.23.4", "os": "linux", "arch": "amd64"}
2024-12-20T14:35:33.284Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.28.15-gke.1388000"}
2024-12-20T14:35:33.285Z	INFO	setup	Registering Components.
2024-12-20T14:35:35.090Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-12-20T14:35:35.093Z	INFO	setup	Starting the Cmd.
2024-12-20T14:35:35.094Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-12-20T14:35:35.094Z	INFO	controller-runtime.metrics	Starting metrics server
2024-12-20T14:35:35.094Z	INFO	controller-runtime.webhook	Starting webhook server
2024-12-20T14:35:35.094Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-12-20T14:35:35.175Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-12-20T14:35:35.175Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-12-20T14:35:35.175Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-12-20T14:35:35.275Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-12-20T14:35:35.288Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-12-20T14:35:35.289Z	DEBUG	events	percona-xtradb-cluster-operator-67cbddd7f9-5rwc4_d544a0b1-b611-4104-9cd8-72d4b43e08f8 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"f1c30618-cc96-42a3-8980-dbeac2c21d22","apiVersion":"coordination.k8s.io/v1","resourceVersion":"36384"}, "reason": "LeaderElection"}
2024-12-20T14:35:35.289Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-12-20T14:35:35.289Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-12-20T14:35:35.289Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-12-20T14:35:35.289Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-12-20T14:35:35.289Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-12-20T14:35:35.289Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-12-20T14:35:35.394Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-12-20T14:35:35.395Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-12-20T14:35:35.395Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-12-20T14:36:18.901Z	INFO	Set CR version	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "version": "1.16.0"}
2024-12-20T14:36:19.378Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-pxc"}
2024-12-20T14:36:19.396Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-proxysql"}
2024-12-20T14:36:19.543Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-proxysql"}
2024-12-20T14:36:19.570Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-pxc"}
2024-12-20T14:36:19.655Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-pxc-unready"}
2024-12-20T14:36:19.751Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-proxysql"}
2024-12-20T14:36:19.845Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "ea16f32a-1282-4550-94f8-203f81d55864", "object": "scaling-proxysql-unready"}
2024-12-20T14:36:20.466Z	DEBUG	Creating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "3dc56af4-c291-46da-b078-b338e78ec1c7", "object": "scaling-pxc"}
2024-12-20T14:37:36.505Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938", "user": "operator"}
2024-12-20T14:37:36.543Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938", "user": "monitor"}
2024-12-20T14:37:36.661Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938"}
2024-12-20T14:37:36.702Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938"}
2024-12-20T14:37:36.739Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938", "user": "xtrabackup"}
2024-12-20T14:37:36.802Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938"}
2024-12-20T14:37:36.842Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938", "user": "replication"}
2024-12-20T14:37:36.877Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2615e4d4-00fa-40b4-92cb-20445a64d938", "err": "get primary pxc pod: not found"}
2024-12-20T14:37:41.416Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "edf375fa-60ef-4752-aa66-eef85bd77c6b", "err": "get primary pxc pod: not found"}
2024-12-20T14:37:46.581Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2d15b4cf-72ba-4baa-b5aa-8bab58a0a49f", "err": "get primary pxc pod: not found"}
2024-12-20T14:37:51.668Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "3edd6da2-272c-4c9b-a327-a37c271d6111", "err": "get primary pxc pod: not found"}
2024-12-20T14:40:01.984Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "613039ca-cf58-4724-b0de-bd89a9159cd2", "user": "root"}
2024-12-20T14:40:02.224Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "613039ca-cf58-4724-b0de-bd89a9159cd2", "new version": "8.0.39-30.1"}
2024-12-20T14:40:04.238Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "613039ca-cf58-4724-b0de-bd89a9159cd2"}
2024-12-20T14:40:09.096Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "6c417fe5-c667-437f-92d2-1a8ccd69da59"}
2024-12-20T14:40:14.348Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2034a6db-4970-467e-9382-5ece0eb090a3"}
2024-12-20T14:40:19.618Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "3a46136d-cf19-4e76-881d-19edd40780c0"}
2024-12-20T14:40:25.079Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "e7162e1d-7b96-4309-bac6-f0da6586c921"}
2024-12-20T14:40:30.244Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "80fe24bd-c2ec-400c-8608-cf1c59c9812e"}
2024-12-20T14:40:35.948Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "6995f277-69b3-4d9a-a1e8-a54cfabd4fc1"}
2024-12-20T14:40:41.194Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "4b74f659-ceab-4952-b68b-a0f8697e4d9b"}
2024-12-20T14:40:46.630Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "48e3e33b-f9f6-4f1e-a367-2c22bafb7ca9"}
2024-12-20T14:40:52.090Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "8abcdbd6-d270-4c78-95c1-89eee9a249f2"}
2024-12-20T14:40:57.200Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "dfb5e835-7d50-41ec-9f3b-4deef60805dc"}
2024-12-20T14:41:02.514Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "478d2c10-3bc0-41d4-973a-e269eac195a1"}
2024-12-20T14:41:07.690Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "7b54c2bc-64d6-4adc-8576-b9993a7661a6"}
2024-12-20T14:41:13.252Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "351058d5-9d01-4eb5-9c68-269202ed4238"}
2024-12-20T14:41:18.313Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "73fb0326-2ec7-4b96-a288-596f81d66e20"}
2024-12-20T14:41:24.114Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "b94403cb-1984-41e3-b9a4-8a49a766a8a2"}
2024-12-20T14:41:25.877Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "26995799-ea0a-472b-875b-1e1a2026691e", "object": "scaling-pxc"}
2024-12-20T14:41:27.808Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "26995799-ea0a-472b-875b-1e1a2026691e"}
2024-12-20T14:44:02.469Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "c6bae0aa-aa6d-43eb-9948-5fceb04e4676"}
2024-12-20T14:44:06.336Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "4f86b16e-df1f-46ee-820a-6cfd84d92100"}
2024-12-20T14:44:11.944Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "432dd09f-cefe-48d6-a636-130e0d944e1b"}
2024-12-20T14:44:17.484Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "7d874d21-a7bb-4a56-93ac-e81ddd505856"}
2024-12-20T14:44:23.124Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "d9660e28-c415-475a-9c21-f7f7e23f9801"}
2024-12-20T14:44:28.265Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "bf87615f-abc0-4cad-85bf-7d0bf7340331"}
2024-12-20T14:44:33.840Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "33a68c17-152a-4d37-bb07-3112d87cabe7"}
2024-12-20T14:44:38.957Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "f01e4fb5-0088-4d54-9732-875bb92c1f9a"}
2024-12-20T14:44:44.668Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "739261cf-80b6-4033-a0be-cdf230570452"}
2024-12-20T14:44:50.005Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "0f8b2197-b56a-4ad5-84f8-c88ebea6e194"}
2024-12-20T14:44:53.175Z	DEBUG	Updating object	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "6fca6746-96d3-4abf-8501-c5e3138b7ce7", "object": "scaling-pxc"}
2024-12-20T14:44:55.251Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "6fca6746-96d3-4abf-8501-c5e3138b7ce7"}
2024-12-20T14:45:04.048Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "eb1ac3d8-db76-4b4d-9a46-702e47fb62f4", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-4: dial tcp: lookup scaling-pxc-4.scaling-pxc.scaling-6084 on 10.186.32.10:53: no such host"}
2024-12-20T14:45:09.359Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "0ac46675-1ef6-42ad-9a6a-17db9420a2c0", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-3: dial tcp: lookup scaling-pxc-3.scaling-pxc.scaling-6084 on 10.186.32.10:53: no such host"}
2024-12-20T14:45:19.833Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "5baff2cb-1d2f-4faf-8e41-cb2f01c47495", "err": "failed to ensure cluster readonly status: connect to pod scaling-pxc-3: dial tcp: lookup scaling-pxc-3.scaling-pxc.scaling-6084 on 10.186.32.10:53: no such host"}
2024-12-20T14:45:27.658Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "0d9ac07a-2010-4f87-8e38-05c5e36e80eb"}
2024-12-20T14:45:32.930Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "scaling-6084", "name": "scaling", "reconcileID": "2e369a4a-a26a-42b8-b9e5-a99ec5e0fe82"}
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl get pxc --all-namespaces -o wide
+ kubectl patch pxc -n scaling-6084 scaling --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/scaling patched (no change)
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.TAZK84s6Rf
++ mktemp
+ local LAST_ERR=/tmp/tmp.frpjxUcOAq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TAZK84s6Rf
perconaxtradbcluster.pxc.percona.com "scaling" deleted
+ cat /tmp/tmp.frpjxUcOAq
+ rm /tmp/tmp.TAZK84s6Rf /tmp/tmp.frpjxUcOAq
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.kQxImAb6P7
++ mktemp
+ local LAST_ERR=/tmp/tmp.HdF1zUilWl
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kQxImAb6P7
No resources found
+ cat /tmp/tmp.HdF1zUilWl
+ rm /tmp/tmp.kQxImAb6P7 /tmp/tmp.HdF1zUilWl
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.gaKczsl3rv
++ mktemp
+ local LAST_ERR=/tmp/tmp.2bz75gXkTH
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gaKczsl3rv
No resources found
+ cat /tmp/tmp.2bz75gXkTH
+ rm /tmp/tmp.gaKczsl3rv /tmp/tmp.2bz75gXkTH
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.3MOOamSmj1
++ mktemp
+ local LAST_ERR=/tmp/tmp.aNffCwT5J9
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3MOOamSmj1
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.aNffCwT5J9
+ rm /tmp/tmp.3MOOamSmj1 /tmp/tmp.aNffCwT5J9
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.16.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ rm -rf /tmp/tmp.XRBAG9Aezm
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
+ kubectl_bin delete --grace-period=0 --force=true namespace scaling-6084
++ mktemp
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.SUIpDJQb5V
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xe2JTINFDQ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace scaling-6084
++ mktemp
+ local LAST_OUT=/tmp/tmp.aTE2JbtD7W
++ mktemp
+ local LAST_ERR=/tmp/tmp.L2a8haFlKi
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator