Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/logs/smart-update2-8-0.log WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 + API=pxc.percona.com/v9-9-9 + TARGET_IMAGE_PXC=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + CLUSTER=smart-update + CLUSTER_SIZE=3 + PROXY_SIZE=2 + [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 == *\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\-\o\p\e\r\a\t\o\r* ]] ++ echo -n perconalab/percona-xtradb-cluster-operator:main-pxc8.0 ++ /usr/bin/sed -r 's/.*([0-9].[0-9])$/\1/' + PXC_VER=8.0 + TARGET_IMAGE_PXC_VS=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + VS_URL=http://version-service + VS_PORT=11000 + VS_ENDPOINT=http://version-service:11000 + main + create_infra smart-update2-20475 + local ns=smart-update2-20475 + '[' -n pxc-operator ']' + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl patch pxc -n smart-update2-10103 minimal-cluster --type=merge -p '{"metadata":{"finalizers":[]}}' perconaxtradbcluster.pxc.percona.com/minimal-cluster patched (no change) + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.HKr9yFISDb ++ mktemp + local LAST_ERR=/tmp/tmp.T0FGy0UG6A + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.HKr9yFISDb perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted + cat /tmp/tmp.T0FGy0UG6A + rm /tmp/tmp.HKr9yFISDb /tmp/tmp.T0FGy0UG6A + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.m3UjEBdFfl ++ mktemp + local LAST_ERR=/tmp/tmp.YVzbu8OMf2 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.m3UjEBdFfl No resources found + cat /tmp/tmp.YVzbu8OMf2 + rm /tmp/tmp.m3UjEBdFfl /tmp/tmp.YVzbu8OMf2 + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.gMNLNALigF ++ mktemp + local LAST_ERR=/tmp/tmp.fx74TvnRqF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gMNLNALigF No resources found + cat /tmp/tmp.fx74TvnRqF + rm /tmp/tmp.gMNLNALigF /tmp/tmp.fx74TvnRqF + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ sed s/NAMESPACE// ++ tail -n1 ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ awk '{print $1}' ++ kubectl api-resources ++ grep chaos-mesh ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + '[' -n '' ']' + desc 'cleaned up old namespaces pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces pxc-operator ----------------------------------------------------------------------------------- + awk '{print$1}' + kubectl_bin delete namespace pxc-operator + xargs kubectl delete ns ++ mktemp + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.9mIjORjycT + kubectl_bin get ns ++ mktemp + local LAST_OUT=/tmp/tmp.SzGV2IGiP0 ++ mktemp + local LAST_ERR=/tmp/tmp.n2Nm93USdM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns ++ mktemp + local LAST_ERR=/tmp/tmp.NZFkFMIH3q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SzGV2IGiP0 + cat /tmp/tmp.n2Nm93USdM + rm /tmp/tmp.SzGV2IGiP0 /tmp/tmp.n2Nm93USdM + return 0 namespace "cert-manager" deleted namespace "smart-update2-10103" deleted Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9mIjORjycT namespace "pxc-operator" deleted + cat /tmp/tmp.NZFkFMIH3q + rm /tmp/tmp.9mIjORjycT /tmp/tmp.NZFkFMIH3q + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + echo -n 'namespace/pxc-operator - ' namespace/pxc-operator - + set +o xtrace Error from server (NotFound): namespaces "pxc-operator" not found + desc 'create namespace pxc-operator' + set +o xtrace ----------------------------------------------------------------------------------- create namespace pxc-operator ----------------------------------------------------------------------------------- + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.7xOZMSBVM5 ++ mktemp + local LAST_ERR=/tmp/tmp.d0K2chbKv5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7xOZMSBVM5 namespace/pxc-operator created + cat /tmp/tmp.d0K2chbKv5 + rm /tmp/tmp.7xOZMSBVM5 /tmp/tmp.d0K2chbKv5 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.NOpr3lt6fW +++ mktemp ++ local LAST_ERR=/tmp/tmp.cO93r2K8ak ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.NOpr3lt6fW ++ cat /tmp/tmp.cO93r2K8ak ++ rm /tmp/tmp.NOpr3lt6fW /tmp/tmp.cO93r2K8ak ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5 --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.qOQOMgOBUN ++ mktemp + local LAST_ERR=/tmp/tmp.VUJTEz0PFZ + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5 --namespace=pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qOQOMgOBUN Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5" modified. + cat /tmp/tmp.VUJTEz0PFZ + rm /tmp/tmp.qOQOMgOBUN /tmp/tmp.VUJTEz0PFZ + return 0 + deploy_operator + desc 'start PXC operator' + set +o xtrace ----------------------------------------------------------------------------------- start PXC operator ----------------------------------------------------------------------------------- + kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.mYfJ0l7S8E ++ mktemp + local LAST_ERR=/tmp/tmp.PkJAkQZEE1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/crd.yaml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.mYfJ0l7S8E customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied + cat /tmp/tmp.PkJAkQZEE1 + rm /tmp/tmp.mYfJ0l7S8E /tmp/tmp.PkJAkQZEE1 + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/cw-rbac.yaml + sed -e 's^namespace: .*^namespace: pxc-operator^' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kxcVpXP8PS ++ mktemp + local LAST_ERR=/tmp/tmp.L6r8NDzGXo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kxcVpXP8PS clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.L6r8NDzGXo + rm /tmp/tmp.kxcVpXP8PS /tmp/tmp.L6r8NDzGXo + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/cw-operator.yaml + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c^' + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' - ++ mktemp + yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' - + local LAST_OUT=/tmp/tmp.RJoQsEIRqG ++ mktemp + local LAST_ERR=/tmp/tmp.dlQY04afZK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.RJoQsEIRqG deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.dlQY04afZK + rm /tmp/tmp.RJoQsEIRqG /tmp/tmp.dlQY04afZK + return 0 + sleep 10 + kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s ++ mktemp + local LAST_OUT=/tmp/tmp.AhX9oICZLT ++ mktemp + local LAST_ERR=/tmp/tmp.NHrwI9tri9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AhX9oICZLT pod/percona-xtradb-cluster-operator-6986c88cb-zpdjm condition met + cat /tmp/tmp.NHrwI9tri9 + rm /tmp/tmp.AhX9oICZLT /tmp/tmp.NHrwI9tri9 + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Q3ukwxzkqM +++ mktemp ++ local LAST_ERR=/tmp/tmp.TtTWK0N40V ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Q3ukwxzkqM ++ cat /tmp/tmp.TtTWK0N40V ++ rm /tmp/tmp.Q3ukwxzkqM /tmp/tmp.TtTWK0N40V ++ return 0 + wait_pod percona-xtradb-cluster-operator-6986c88cb-zpdjm 480 pxc-operator + local pod=percona-xtradb-cluster-operator-6986c88cb-zpdjm + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-6986c88cb-zpdjm ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-6986c88cb-zpdjm condition met percona-xtradb-cluster-operator-6986c88cb-zpdjm.Ok + sleep 3 + create_namespace smart-update2-20475 + local namespace=smart-update2-20475 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ sed s/NAMESPACE// ++ awk '-F ' '{print $2}' + local chaos_mesh_ns= + '[' -n '' ']' ++ kubectl get MutatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete MutatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl get ValidatingWebhookConfiguration ++ grep validate-auth ++ awk '{print $1}' + timeout 30 kubectl delete ValidatingWebhookConfiguration error: resource(s) were provided, but no name was specified + : ++ kubectl api-resources ++ grep chaos-mesh ++ awk '{print $1}' ++ kubectl get crd ++ grep chaos-mesh.org ++ awk '{print $1}' + timeout 30 kubectl delete crd error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrolebinding ++ grep chaos-mesh ++ awk '{print $1}' + timeout 30 kubectl delete clusterrolebinding error: resource(s) were provided, but no name was specified + : ++ kubectl get clusterrole ++ awk '{print $1}' ++ grep chaos-mesh + timeout 30 kubectl delete clusterrole error: resource(s) were provided, but no name was specified + : + desc 'cleaned up all old namespaces' + set +o xtrace ----------------------------------------------------------------------------------- cleaned up all old namespaces ----------------------------------------------------------------------------------- + kubectl_bin get ns + egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + '[' -n '' ']' + desc 'cleaned up old namespaces smart-update2-20475' + xargs kubectl delete ns + set +o xtrace ----------------------------------------------------------------------------------- cleaned up old namespaces smart-update2-20475 ----------------------------------------------------------------------------------- + kubectl_bin delete namespace smart-update2-20475 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.Od8Ow2YXMs ++ mktemp + local LAST_OUT=/tmp/tmp.9m2VhkmPdW ++ mktemp + local LAST_ERR=/tmp/tmp.hrdnRQPYkL + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.X76n7F5XZs + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get ns + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-20475 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Od8Ow2YXMs + cat /tmp/tmp.hrdnRQPYkL + rm /tmp/tmp.Od8Ow2YXMs /tmp/tmp.hrdnRQPYkL + return 0 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl delete namespace smart-update2-20475 Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.9m2VhkmPdW + cat /tmp/tmp.X76n7F5XZs Error from server (NotFound): namespaces "smart-update2-20475" not found + rm /tmp/tmp.9m2VhkmPdW /tmp/tmp.X76n7F5XZs + return 1 + : + wait_for_delete namespace/smart-update2-20475 + local res=namespace/smart-update2-20475 + echo -n 'namespace/smart-update2-20475 - ' namespace/smart-update2-20475 - + set +o xtrace Error from server (NotFound): namespaces "smart-update2-20475" not found + desc 'create namespace smart-update2-20475' + set +o xtrace ----------------------------------------------------------------------------------- create namespace smart-update2-20475 ----------------------------------------------------------------------------------- + kubectl_bin create namespace smart-update2-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.UwWJYbRorw ++ mktemp + local LAST_ERR=/tmp/tmp.uGYcTH5cSw + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace smart-update2-20475 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.UwWJYbRorw namespace/smart-update2-20475 created + cat /tmp/tmp.uGYcTH5cSw + rm /tmp/tmp.UwWJYbRorw /tmp/tmp.uGYcTH5cSw + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.iuqnpPfYr1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZR476TNFOJ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl config current-context ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.iuqnpPfYr1 ++ cat /tmp/tmp.ZR476TNFOJ ++ rm /tmp/tmp.iuqnpPfYr1 /tmp/tmp.ZR476TNFOJ ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5 --namespace=smart-update2-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.6CqF82CBvG ++ mktemp + local LAST_ERR=/tmp/tmp.lNUzN1gxb6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5 --namespace=smart-update2-20475 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.6CqF82CBvG Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1714-41f82f9c-1-cluster5" modified. + cat /tmp/tmp.lNUzN1gxb6 + rm /tmp/tmp.6CqF82CBvG /tmp/tmp.lNUzN1gxb6 + return 0 + apply_secrets + desc 'create secrets for cloud storages' + set +o xtrace ----------------------------------------------------------------------------------- create secrets for cloud storages ----------------------------------------------------------------------------------- + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.7PMtAZtLcq ++ mktemp + local LAST_ERR=/tmp/tmp.mULzqzZIu4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/cloud-secret.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.7PMtAZtLcq secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.mULzqzZIu4 + rm /tmp/tmp.7PMtAZtLcq /tmp/tmp.mULzqzZIu4 + return 0 + deploy_version_service + desc 'install version service' + set +o xtrace ----------------------------------------------------------------------------------- install version service ----------------------------------------------------------------------------------- + kubectl_bin create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json ++ mktemp + local LAST_OUT=/tmp/tmp.oVo46YOGWw ++ mktemp + local LAST_ERR=/tmp/tmp.w5IVd0V4up + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create configmap versions --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.dep.json --from-file /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/operator.9.9.9.pxc-operator.json + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.oVo46YOGWw configmap/versions created + cat /tmp/tmp.w5IVd0V4up + rm /tmp/tmp.oVo46YOGWw /tmp/tmp.w5IVd0V4up + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.f1689PpTDg ++ mktemp + local LAST_ERR=/tmp/tmp.zOFwkzmUQr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.f1689PpTDg deployment.apps/version-service created service/version-service created + cat /tmp/tmp.zOFwkzmUQr + rm /tmp/tmp.f1689PpTDg /tmp/tmp.zOFwkzmUQr + return 0 + sleep 10 + deploy_cert_manager + desc 'deploy cert manager' + set +o xtrace ----------------------------------------------------------------------------------- deploy cert manager ----------------------------------------------------------------------------------- + kubectl_bin create namespace cert-manager ++ mktemp + local LAST_OUT=/tmp/tmp.NTKxjpId9T ++ mktemp + local LAST_ERR=/tmp/tmp.agvKFzAvbY + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl create namespace cert-manager + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NTKxjpId9T namespace/cert-manager created + cat /tmp/tmp.agvKFzAvbY + rm /tmp/tmp.NTKxjpId9T /tmp/tmp.agvKFzAvbY + return 0 + kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true ++ mktemp + local LAST_OUT=/tmp/tmp.yVDW8IRone ++ mktemp + local LAST_ERR=/tmp/tmp.Kg56mqpY7G + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.yVDW8IRone namespace/cert-manager labeled + cat /tmp/tmp.Kg56mqpY7G + rm /tmp/tmp.yVDW8IRone /tmp/tmp.Kg56mqpY7G + return 0 + kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false ++ mktemp + local LAST_OUT=/tmp/tmp.gptCJMsbYA ++ mktemp + local LAST_ERR=/tmp/tmp.AKX1EcbXJB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml --validate=false + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.gptCJMsbYA namespace/cert-manager configured customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged serviceaccount/cert-manager-cainjector created serviceaccount/cert-manager created serviceaccount/cert-manager-webhook created clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews configured role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection configured rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created service/cert-manager created service/cert-manager-webhook created deployment.apps/cert-manager-cainjector created deployment.apps/cert-manager created deployment.apps/cert-manager-webhook created mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured + cat /tmp/tmp.AKX1EcbXJB Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.gptCJMsbYA /tmp/tmp.AKX1EcbXJB + return 0 + '[' '' == 4.10 ']' + sleep 70 ++ grep :8.0 ++ head -n1 ++ sort -V ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ tail -n3 +++ get_operator_pod +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Xp118kOjdh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KsHXiQxozt +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Xp118kOjdh +++ cat /tmp/tmp.KsHXiQxozt +++ rm /tmp/tmp.Xp118kOjdh /tmp/tmp.KsHXiQxozt +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-6986c88cb-zpdjm -n pxc-operator -- curl -s http://version-service.smart-update2-20475.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.30i99XPaGv +++ mktemp ++ local LAST_ERR=/tmp/tmp.wFR968xfZe ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-6986c88cb-zpdjm -n pxc-operator -- curl -s http://version-service.smart-update2-20475.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.30i99XPaGv ++ cat /tmp/tmp.wFR968xfZe Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.30i99XPaGv /tmp/tmp.wFR968xfZe ++ return 0 + IMAGE_PXC=percona/percona-xtradb-cluster:8.0.19-10.1 + kubectl_bin patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' ++ mktemp + local LAST_OUT=/tmp/tmp.uS8A5vUIZC ++ mktemp + local LAST_ERR=/tmp/tmp.fINxWD6DIx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch crd perconaxtradbclusters.pxc.percona.com --type=json -p '[{"op":"add","path":"/spec/versions/-", "value":{"name": "v9-9-9","schema": {"openAPIV3Schema": {"properties": {"spec": {"type": "object","x-kubernetes-preserve-unknown-fields": true},"status": {"type": "object", "x-kubernetes-preserve-unknown-fields": true}}, "type": "object" }}, "served": true, "storage": false, "subresources": { "status": {}}}}]' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uS8A5vUIZC customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com patched + cat /tmp/tmp.fINxWD6DIx + rm /tmp/tmp.uS8A5vUIZC /tmp/tmp.fINxWD6DIx + return 0 + kubectl_bin -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-20475.svc.cluster.local:11000 ++ mktemp + local LAST_OUT=/tmp/tmp.149Ktig1wi ++ mktemp + local LAST_ERR=/tmp/tmp.6vTwt5aoXR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl -n pxc-operator set env deploy/percona-xtradb-cluster-operator PERCONA_VS_FALLBACK_URI=http://version-service.smart-update2-20475.svc.cluster.local:11000 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.149Ktig1wi deployment.apps/percona-xtradb-cluster-operator env updated + cat /tmp/tmp.6vTwt5aoXR + rm /tmp/tmp.149Ktig1wi /tmp/tmp.6vTwt5aoXR + return 0 + desc 'Starting telemetry testing' + set +o xtrace ----------------------------------------------------------------------------------- Starting telemetry testing ----------------------------------------------------------------------------------- + /usr/bin/sed s/version-service/version-service-cr/g /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml ++ yq 'select(.kind == "Deployment").spec.template.spec.containers[0].image' /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml + kubectl_bin apply -f - + yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image) = "perconalab/version-service:main-e378a19"' ++ mktemp + local LAST_OUT=/tmp/tmp.kE0CYCcBn0 ++ mktemp + local LAST_ERR=/tmp/tmp.xxzAKp8CkU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kE0CYCcBn0 deployment.apps/version-service-cr created service/version-service-cr created + cat /tmp/tmp.xxzAKp8CkU + rm /tmp/tmp.kE0CYCcBn0 /tmp/tmp.xxzAKp8CkU + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.T8IJbQZRFk ++ mktemp + local LAST_ERR=/tmp/tmp.xAtybjkdvR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.T8IJbQZRFk pod "version-service-6d49c56954-dwg44" deleted + cat /tmp/tmp.xAtybjkdvR + rm /tmp/tmp.T8IJbQZRFk /tmp/tmp.xAtybjkdvR + return 0 ++ echo -n percona/percona-xtradb-cluster:8.0.19-10.1 ++ sed -r 's/^.*:([0-9]+.[0-9]+).*/\1/' + IMAGE_PREFIX=8.0 + desc 'Enable telemetry on operator level' + set +o xtrace ----------------------------------------------------------------------------------- Enable telemetry on operator level ----------------------------------------------------------------------------------- + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "false"' + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.W5nfvNSEIg ++ mktemp + local LAST_ERR=/tmp/tmp.27XdBovpT9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + kubectl_bin apply -n pxc-operator -f - ++ mktemp + local LAST_OUT=/tmp/tmp.NBAh2H0q2q ++ mktemp + local LAST_ERR=/tmp/tmp.x5YHXtv5BR + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.W5nfvNSEIg + cat /tmp/tmp.27XdBovpT9 + rm /tmp/tmp.W5nfvNSEIg /tmp/tmp.27XdBovpT9 + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NBAh2H0q2q deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.x5YHXtv5BR + rm /tmp/tmp.NBAh2H0q2q /tmp/tmp.x5YHXtv5BR + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.Eoxruf8IbH +++ mktemp ++ local LAST_ERR=/tmp/tmp.DdAD5cTSjL ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Eoxruf8IbH ++ cat /tmp/tmp.DdAD5cTSjL ++ rm /tmp/tmp.Eoxruf8IbH /tmp/tmp.DdAD5cTSjL ++ return 0 + wait_pod percona-xtradb-cluster-operator-5f545d765c-zhzmj 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5f545d765c-zhzmj + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5f545d765c-zhzmj ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-5f545d765c-zhzmj condition met percona-xtradb-cluster-operator-5f545d765c-zhzmj.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 disabled enabled + local cr_vs_uri=http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=enabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4P3QtY8ECn ++ mktemp + local LAST_ERR=/tmp/tmp.aDeid5Z3sV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4P3QtY8ECn deployment.apps/pxc-client created + cat /tmp/tmp.aDeid5Z3sV + rm /tmp/tmp.4P3QtY8ECn /tmp/tmp.aDeid5Z3sV + return 0 + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.hVWvGsHxE0 ++ mktemp + local LAST_ERR=/tmp/tmp.nKQkmbDXNC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hVWvGsHxE0 secret/minimal-cluster created secret/some-name-ssl created secret/some-name-ssl-internal created + cat /tmp/tmp.nKQkmbDXNC + rm /tmp/tmp.hVWvGsHxE0 /tmp/tmp.nKQkmbDXNC + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-20475.svc.cluster.local:11000"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' + kubectl_bin apply -f - + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' ++ mktemp + local LAST_OUT=/tmp/tmp.bhqKXxQl5a ++ mktemp + local LAST_ERR=/tmp/tmp.zN5m0UAhfV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + yq eval '.spec.upgradeOptions.apply="disabled"' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.bhqKXxQl5a perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.zN5m0UAhfV + rm /tmp/tmp.bhqKXxQl5a /tmp/tmp.zN5m0UAhfV + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo minimal-cluster-pxc-0 + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met minimal-cluster-pxc-0.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AwORaze7Rk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9IlbgBMhqi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.AwORaze7Rk +++ cat /tmp/tmp.9IlbgBMhqi +++ rm /tmp/tmp.AwORaze7Rk /tmp/tmp.9IlbgBMhqi +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RJeHxA0X51 +++ mktemp ++ local LAST_ERR=/tmp/tmp.mt80LmSTDU ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.RJeHxA0X51 ++ cat /tmp/tmp.mt80LmSTDU ++ rm /tmp/tmp.RJeHxA0X51 /tmp/tmp.mt80LmSTDU ++ return 0 + client_pod=pxc-client-fdcf9cfbb-k6nmb + wait_pod pxc-client-fdcf9cfbb-k6nmb + local pod=pxc-client-fdcf9cfbb-k6nmb + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-fdcf9cfbb-k6nmb ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-k6nmb condition met pxc-client-fdcf9cfbb-k6nmb.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ilX7cnQSrv +++ mktemp ++ local LAST_ERR=/tmp/tmp.uSTc1iY3jb ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ilX7cnQSrv ++ cat /tmp/tmp.uSTc1iY3jb ++ rm /tmp/tmp.ilX7cnQSrv /tmp/tmp.uSTc1iY3jb ++ return 0 + client_pod=pxc-client-fdcf9cfbb-k6nmb + wait_pod pxc-client-fdcf9cfbb-k6nmb + local pod=pxc-client-fdcf9cfbb-k6nmb + local max_retry=480 + local ns= ++ echo pxc-client-fdcf9cfbb-k6nmb ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-k6nmb condition met pxc-client-fdcf9cfbb-k6nmb.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + grep -E 'server request payload|unary call' + jq 'del(."grpc.time_ms")' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.start_time")' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-77d9b658bb-sw966 ++ mktemp + local LAST_OUT=/tmp/tmp.4NWAmZ8cwV ++ mktemp + local LAST_ERR=/tmp/tmp.3KszzGj909 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-77d9b658bb-sw966 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.4NWAmZ8cwV + cat /tmp/tmp.3KszzGj909 + rm /tmp/tmp.4NWAmZ8cwV /tmp/tmp.3KszzGj909 + return 0 + grep -E 'server request payload|unary call' + grep -Eo '\{.*\}' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.start_time")' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + kubectl_bin logs version-service-6d49c56954-d6cph ++ mktemp + local LAST_OUT=/tmp/tmp.zUbkGJfnUb ++ mktemp + local LAST_ERR=/tmp/tmp.y8lEPqLcWn + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-6d49c56954-d6cph + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.zUbkGJfnUb + cat /tmp/tmp.y8lEPqLcWn + rm /tmp/tmp.zUbkGJfnUb /tmp/tmp.y8lEPqLcWn + return 0 + local telemetry_log_file=enabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == enabled ']' + desc 'operator fallback VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator fallback VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.qA2QIzYgTk/enabled_telemetry.version-service.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/enabled_telemetry.version-service-cw.log.json + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.qA2QIzYgTk/enabled_telemetry.version-service-cr.log.json ]] + local image_prefix=disabled + local telemetry_cr_log_file=enabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' disabled == disabled-recommended -a enabled == disabled ']' + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a enabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.2p7K9B9H6V ++ mktemp + local LAST_ERR=/tmp/tmp.4TplEhj6UV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.2p7K9B9H6V perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.4TplEhj6UV + rm /tmp/tmp.2p7K9B9H6V /tmp/tmp.4TplEhj6UV + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.tlp0PRbIQ9 +++ mktemp ++ local LAST_ERR=/tmp/tmp.yCiwpOim12 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tlp0PRbIQ9 ++ cat /tmp/tmp.yCiwpOim12 ++ rm /tmp/tmp.tlp0PRbIQ9 /tmp/tmp.yCiwpOim12 ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-5f545d765c-zhzmj ++ mktemp + local LAST_OUT=/tmp/tmp.YGCVfkhQXt ++ mktemp + local LAST_ERR=/tmp/tmp.xLiDFzCmYm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-5f545d765c-zhzmj + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YGCVfkhQXt pod "percona-xtradb-cluster-operator-5f545d765c-zhzmj" deleted + cat /tmp/tmp.xLiDFzCmYm + rm /tmp/tmp.YGCVfkhQXt /tmp/tmp.xLiDFzCmYm + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.vTyhwehHKC ++ mktemp + local LAST_ERR=/tmp/tmp.ke43Q93APe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.vTyhwehHKC perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted + cat /tmp/tmp.ke43Q93APe + rm /tmp/tmp.vTyhwehHKC /tmp/tmp.ke43Q93APe + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.0lr0wbJLWc ++ mktemp + local LAST_ERR=/tmp/tmp.WOZuTCZn26 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.0lr0wbJLWc deployment.apps "pxc-client" deleted + cat /tmp/tmp.WOZuTCZn26 + rm /tmp/tmp.0lr0wbJLWc /tmp/tmp.WOZuTCZn26 + return 0 + sleep 30 + desc 'Disabling telemetry on the operator level' + set +o xtrace ----------------------------------------------------------------------------------- Disabling telemetry on the operator level ----------------------------------------------------------------------------------- + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.Ir88VDj4gM ++ mktemp + local LAST_ERR=/tmp/tmp.SXxB9SZxH4 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Ir88VDj4gM pod "version-service-cr-77d9b658bb-sw966" deleted + cat /tmp/tmp.SXxB9SZxH4 + rm /tmp/tmp.Ir88VDj4gM /tmp/tmp.SXxB9SZxH4 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.uIj4FCyqk6 ++ mktemp + local LAST_ERR=/tmp/tmp.QLy3Uzajle + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.uIj4FCyqk6 pod "version-service-6d49c56954-d6cph" deleted + cat /tmp/tmp.QLy3Uzajle + rm /tmp/tmp.uIj4FCyqk6 /tmp/tmp.QLy3Uzajle + return 0 + yq '(.spec.template.spec.containers[0].env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' + kubectl_bin apply -n pxc-operator -f - + kubectl_bin get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.327WzZUoZD ++ mktemp + local LAST_OUT=/tmp/tmp.k20bhhk9xF + local LAST_ERR=/tmp/tmp.Kkn71xLn3N + local exit_status=0 ++ mktemp + local LAST_ERR=/tmp/tmp.rOh6uy5uhC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -n pxc-operator -f - ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl get deployment/percona-xtradb-cluster-operator -o yaml -n pxc-operator + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.k20bhhk9xF + cat /tmp/tmp.rOh6uy5uhC + rm /tmp/tmp.k20bhhk9xF /tmp/tmp.rOh6uy5uhC + return 0 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.327WzZUoZD deployment.apps/percona-xtradb-cluster-operator configured + cat /tmp/tmp.Kkn71xLn3N + rm /tmp/tmp.327WzZUoZD /tmp/tmp.Kkn71xLn3N + return 0 + sleep 30 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.1F5k3ykGYd +++ mktemp ++ local LAST_ERR=/tmp/tmp.2q36FJlMKw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1F5k3ykGYd ++ cat /tmp/tmp.2q36FJlMKw ++ rm /tmp/tmp.1F5k3ykGYd /tmp/tmp.2q36FJlMKw ++ return 0 + wait_pod percona-xtradb-cluster-operator-75d57fb4f6-8zdnb 480 pxc-operator + local pod=percona-xtradb-cluster-operator-75d57fb4f6-8zdnb + local max_retry=480 + local ns=pxc-operator ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo percona-xtradb-cluster-operator-75d57fb4f6-8zdnb ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/percona-xtradb-cluster-operator-75d57fb4f6-8zdnb condition met percona-xtradb-cluster-operator-75d57fb4f6-8zdnb.Ok + check_telemetry_transfer http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 8.0-recommended disabled + local cr_vs_uri=http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 + local cr_vs_channel=8.0-recommended + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xUN7eyngvO ++ mktemp + local LAST_ERR=/tmp/tmp.GgTf3OjVQX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xUN7eyngvO deployment.apps/pxc-client created + cat /tmp/tmp.GgTf3OjVQX + rm /tmp/tmp.xUN7eyngvO /tmp/tmp.GgTf3OjVQX + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.JlwiopxT7i ++ mktemp + local LAST_ERR=/tmp/tmp.xWLchi6T2w + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JlwiopxT7i secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.xWLchi6T2w + rm /tmp/tmp.JlwiopxT7i /tmp/tmp.xWLchi6T2w + return 0 + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-20475.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="8.0-recommended"' + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/cr-minimal.yaml + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.crVersion="9.9.9"' + kubectl_bin apply -f - + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' ++ mktemp + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c"' + local LAST_OUT=/tmp/tmp.pyUmdxPAC1 ++ mktemp + local LAST_ERR=/tmp/tmp.cW5PJ906O9 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.pyUmdxPAC1 perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.cW5PJ906O9 + rm /tmp/tmp.pyUmdxPAC1 /tmp/tmp.cW5PJ906O9 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met minimal-cluster-pxc-0.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pWiFNjehOc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sv88BYkKiJ +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.pWiFNjehOc +++ cat /tmp/tmp.sv88BYkKiJ +++ rm /tmp/tmp.pWiFNjehOc /tmp/tmp.sv88BYkKiJ +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BBKJZEzFCX +++ mktemp ++ local LAST_ERR=/tmp/tmp.sZtZeypLhy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BBKJZEzFCX ++ cat /tmp/tmp.sZtZeypLhy ++ rm /tmp/tmp.BBKJZEzFCX /tmp/tmp.sZtZeypLhy ++ return 0 + client_pod=pxc-client-fdcf9cfbb-rd58f + wait_pod pxc-client-fdcf9cfbb-rd58f + local pod=pxc-client-fdcf9cfbb-rd58f + local max_retry=480 + local ns= ++ echo pxc-client-fdcf9cfbb-rd58f ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-rd58f condition met pxc-client-fdcf9cfbb-rd58f.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JxU2w8fU0Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.m1sbqOR1je ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JxU2w8fU0Z ++ cat /tmp/tmp.m1sbqOR1je ++ rm /tmp/tmp.JxU2w8fU0Z /tmp/tmp.m1sbqOR1je ++ return 0 + client_pod=pxc-client-fdcf9cfbb-rd58f + wait_pod pxc-client-fdcf9cfbb-rd58f + local pod=pxc-client-fdcf9cfbb-rd58f + local max_retry=480 + local ns= ++ echo pxc-client-fdcf9cfbb-rd58f ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-rd58f condition met pxc-client-fdcf9cfbb-rd58f.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + grep -E 'server request payload|unary call' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-77d9b658bb-6zgtt ++ mktemp + local LAST_OUT=/tmp/tmp.wR4w6YUsvW ++ mktemp + local LAST_ERR=/tmp/tmp.MLUS5ZWFKd + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-77d9b658bb-6zgtt + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.wR4w6YUsvW + cat /tmp/tmp.MLUS5ZWFKd + rm /tmp/tmp.wR4w6YUsvW /tmp/tmp.MLUS5ZWFKd + return 0 + grep -E 'server request payload|unary call' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + grep -Eo '\{.*\}' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-6d49c56954-bmdnj ++ mktemp + local LAST_OUT=/tmp/tmp.QMwStt1odz ++ mktemp + local LAST_ERR=/tmp/tmp.cC4tvCoYbC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-6d49c56954-bmdnj + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.QMwStt1odz + cat /tmp/tmp.cC4tvCoYbC + rm /tmp/tmp.QMwStt1odz /tmp/tmp.cC4tvCoYbC + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == enabled ']' + local image_prefix=8.0 + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' 8.0-recommended == 8.0-recommended -a disabled == disabled ']' + desc 'cr VS should have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- cr VS should have telemetry ----------------------------------------------------------------------------------- + diff /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json /dev/fd/63 ++ grep -f /tmp/tmp.qA2QIzYgTk/disabled_telemetry.version-service-cr.log.json /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/disabled_telemetry.version-service-cr-8.0-cw.log.json + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.qA2QIzYgTk/disabled_telemetry.version-service.log.json ]] + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' 8.0-recommended == disabled -a disabled == disabled ']' + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.qgndYy2kGf ++ mktemp + local LAST_ERR=/tmp/tmp.A7CwS9LucU + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.qgndYy2kGf perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.A7CwS9LucU + rm /tmp/tmp.qgndYy2kGf /tmp/tmp.A7CwS9LucU + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.X8kWGonQ71 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DQhdUlK0r5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.X8kWGonQ71 ++ cat /tmp/tmp.DQhdUlK0r5 ++ rm /tmp/tmp.X8kWGonQ71 /tmp/tmp.DQhdUlK0r5 ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-8zdnb ++ mktemp + local LAST_OUT=/tmp/tmp.n6ioLDzRRm ++ mktemp + local LAST_ERR=/tmp/tmp.zE0dAVsbQa + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-8zdnb + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.n6ioLDzRRm pod "percona-xtradb-cluster-operator-75d57fb4f6-8zdnb" deleted + cat /tmp/tmp.zE0dAVsbQa + rm /tmp/tmp.n6ioLDzRRm /tmp/tmp.zE0dAVsbQa + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.kJfPZmLKaY ++ mktemp + local LAST_ERR=/tmp/tmp.NS3bfXG5pb + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.kJfPZmLKaY perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted + cat /tmp/tmp.NS3bfXG5pb + rm /tmp/tmp.kJfPZmLKaY /tmp/tmp.NS3bfXG5pb + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.C3a6yKe92I ++ mktemp + local LAST_ERR=/tmp/tmp.6nGYhdjbXl + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.C3a6yKe92I deployment.apps "pxc-client" deleted + cat /tmp/tmp.6nGYhdjbXl + rm /tmp/tmp.C3a6yKe92I /tmp/tmp.6nGYhdjbXl + return 0 + sleep 30 + kubectl_bin delete pod -l run=version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.CFbuky0svI ++ mktemp + local LAST_ERR=/tmp/tmp.RU2qHu6ee3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.CFbuky0svI pod "version-service-cr-77d9b658bb-6zgtt" deleted + cat /tmp/tmp.RU2qHu6ee3 + rm /tmp/tmp.CFbuky0svI /tmp/tmp.RU2qHu6ee3 + return 0 + kubectl_bin delete pod -l run=version-service ++ mktemp + local LAST_OUT=/tmp/tmp.fwjNtduyJm ++ mktemp + local LAST_ERR=/tmp/tmp.HPbUrZNBDp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -l run=version-service + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fwjNtduyJm pod "version-service-6d49c56954-bmdnj" deleted + cat /tmp/tmp.HPbUrZNBDp + rm /tmp/tmp.fwjNtduyJm /tmp/tmp.HPbUrZNBDp + return 0 + check_telemetry_transfer http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 disabled disabled + local cr_vs_uri=http://version-service-cr.smart-update2-20475.svc.cluster.local:11000 + local cr_vs_channel=disabled + local telemetry_state=disabled + desc 'create PXC minimal cluster' + set +o xtrace ----------------------------------------------------------------------------------- create PXC minimal cluster ----------------------------------------------------------------------------------- + cluster=minimal-cluster + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.rd6QLFOm5V ++ mktemp + local LAST_ERR=/tmp/tmp.EfGZfUmC4X + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rd6QLFOm5V deployment.apps/pxc-client created + cat /tmp/tmp.EfGZfUmC4X + rm /tmp/tmp.rd6QLFOm5V /tmp/tmp.EfGZfUmC4X + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + yq eval '(. | select(.metadata.name == "my-cluster-secrets") | .metadata.name) = "minimal-cluster"' + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tJVl14yHa4 ++ mktemp + local LAST_ERR=/tmp/tmp.KWi55eL3u5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.tJVl14yHa4 secret/minimal-cluster unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.KWi55eL3u5 + rm /tmp/tmp.tJVl14yHa4 /tmp/tmp.KWi55eL3u5 + return 0 + yq /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/deploy/cr-minimal.yaml + yq eval '.spec.upgradeOptions.versionServiceEndpoint="http://version-service-cr.smart-update2-20475.svc.cluster.local:11000"' + yq eval '.spec.upgradeOptions.apply="disabled"' + yq eval '.spec.haproxy.image="perconalab/percona-xtradb-cluster-operator:main-haproxy"' + yq eval '.spec.crVersion="9.9.9"' + yq eval '.spec.logcollector.image="perconalab/percona-xtradb-cluster-operator:main-logcollector"' + kubectl_bin apply -f - ++ mktemp + yq eval '.spec.pxc.image="percona/percona-xtradb-cluster:8.0.19-10.1"' + yq eval '.spec.initContainer.image="perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c"' + local LAST_OUT=/tmp/tmp.GD6NVaPZcB ++ mktemp + local LAST_ERR=/tmp/tmp.W99Ydp1Bw7 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.GD6NVaPZcB perconaxtradbcluster.pxc.percona.com/minimal-cluster created + cat /tmp/tmp.W99Ydp1Bw7 + rm /tmp/tmp.GD6NVaPZcB /tmp/tmp.W99Ydp1Bw7 + return 0 + desc 'check if Pod is started' + set +o xtrace ----------------------------------------------------------------------------------- check if Pod is started ----------------------------------------------------------------------------------- + wait_for_running minimal-cluster-pxc 1 + local name=minimal-cluster-pxc + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-pxc-0 480 + local pod=minimal-cluster-pxc-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-pxc-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/minimal-cluster-pxc-0 condition met minimal-cluster-pxc-0.Ok + sleep 20 + local proxy ++ get_proxy minimal-cluster ++ local target_cluster=minimal-cluster +++ kubectl_bin get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vlPJ1cnhPc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.si5nxl70MW +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc minimal-cluster -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.vlPJ1cnhPc +++ cat /tmp/tmp.si5nxl70MW +++ rm /tmp/tmp.vlPJ1cnhPc /tmp/tmp.si5nxl70MW +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo minimal-cluster-haproxy ++ return + proxy=minimal-cluster-haproxy + wait_for_running minimal-cluster-haproxy 1 + local name=minimal-cluster-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod minimal-cluster-haproxy-0 480 + local pod=minimal-cluster-haproxy-0 + local max_retry=480 + local ns= ++ echo minimal-cluster-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/minimal-cluster-haproxy-0 condition met minimal-cluster-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Jj53QdP5Kj +++ mktemp ++ local LAST_ERR=/tmp/tmp.eTnU8W7WCI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.Jj53QdP5Kj ++ cat /tmp/tmp.eTnU8W7WCI ++ rm /tmp/tmp.Jj53QdP5Kj /tmp/tmp.eTnU8W7WCI ++ return 0 + client_pod=pxc-client-fdcf9cfbb-xqkn7 + wait_pod pxc-client-fdcf9cfbb-xqkn7 + local pod=pxc-client-fdcf9cfbb-xqkn7 + local max_retry=480 + local ns= ++ echo pxc-client-fdcf9cfbb-xqkn7 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-xqkn7 condition met pxc-client-fdcf9cfbb-xqkn7.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h minimal-cluster-haproxy -uroot -proot_password -P' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h minimal-cluster-haproxy -uroot -proot_password -P' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.thyiKhX3cC +++ mktemp ++ local LAST_ERR=/tmp/tmp.RHXS22mzhI ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.thyiKhX3cC ++ cat /tmp/tmp.RHXS22mzhI ++ rm /tmp/tmp.thyiKhX3cC /tmp/tmp.RHXS22mzhI ++ return 0 + client_pod=pxc-client-fdcf9cfbb-xqkn7 + wait_pod pxc-client-fdcf9cfbb-xqkn7 + local pod=pxc-client-fdcf9cfbb-xqkn7 + local max_retry=480 + local ns= ++ echo pxc-client-fdcf9cfbb-xqkn7 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-fdcf9cfbb-xqkn7 condition met pxc-client-fdcf9cfbb-xqkn7.Ok + set +o xtrace [ERROR] mysql: option '-P' requires an argument command terminated with exit code 5 + grep -E 'server request payload|unary call' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + jq 'del(."grpc.time_ms")' + grep -Eo '\{.*\}' + jq 'del(."grpc.request.content".msg.kubeVersion)' ++ kubectl get pods --selector=run=version-service-cr -o 'jsonpath={.items[0].metadata.name}' + kubectl_bin logs version-service-cr-77d9b658bb-66clb ++ mktemp + local LAST_OUT=/tmp/tmp.456NmlOQEa ++ mktemp + local LAST_ERR=/tmp/tmp.zfluhsaHg5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-cr-77d9b658bb-66clb + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.456NmlOQEa + cat /tmp/tmp.zfluhsaHg5 + rm /tmp/tmp.456NmlOQEa /tmp/tmp.zfluhsaHg5 + return 0 + grep -Eo '\{.*\}' + grep -E 'server request payload|unary call' ++ kubectl get pods --selector=run=version-service -o 'jsonpath={.items[0].metadata.name}' + jq 'del(."grpc.request.content".msg.kubeVersion)' + jq 'del(."grpc.time_ms")' + jq 'del(."grpc.start_time")' + jq 'del(."grpc.request.content".msg.customResourceUid)' + kubectl_bin logs version-service-6d49c56954-q45zk ++ mktemp + local LAST_OUT=/tmp/tmp.9mEWqUFQzX ++ mktemp + local LAST_ERR=/tmp/tmp.sULObhwSFL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs version-service-6d49c56954-q45zk + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.9mEWqUFQzX + cat /tmp/tmp.sULObhwSFL + rm /tmp/tmp.9mEWqUFQzX /tmp/tmp.sULObhwSFL + return 0 + local telemetry_log_file=disabled_telemetry.version-service-cw.log.json + desc 'telemetry was disabled in CR but in operator not' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR but in operator not ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == enabled ']' + local image_prefix=disabled + local telemetry_cr_log_file=disabled_telemetry.version-service-cr-disabled-cw.log.json + desc 'telemetry was disabled in operator but not in CR' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in operator but not in CR ----------------------------------------------------------------------------------- + '[' disabled == disabled-recommended -a disabled == disabled ']' + desc 'telemetry was disabled in CR as well as in operator' + set +o xtrace ----------------------------------------------------------------------------------- telemetry was disabled in CR as well as in operator ----------------------------------------------------------------------------------- + '[' disabled == disabled -a disabled == disabled ']' + desc 'CR VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- CR VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.qA2QIzYgTk/disabled_telemetry.version-service-cr.log.json ]] + desc 'operator VS should not have telemetry' + set +o xtrace ----------------------------------------------------------------------------------- operator VS should not have telemetry ----------------------------------------------------------------------------------- + [[ -s /tmp/tmp.qA2QIzYgTk/disabled_telemetry.version-service.log.json ]] + kubectl_bin patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' ++ mktemp + local LAST_OUT=/tmp/tmp.AflYNeMeGe ++ mktemp + local LAST_ERR=/tmp/tmp.bDcZLX4bJt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc minimal-cluster --type=merge -p '{"metadata":{"finalizers":["delete-pxc-pvc"]}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.AflYNeMeGe perconaxtradbcluster.pxc.percona.com/minimal-cluster patched + cat /tmp/tmp.bDcZLX4bJt + rm /tmp/tmp.AflYNeMeGe /tmp/tmp.bDcZLX4bJt + return 0 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.wBUDnmLohy +++ mktemp ++ local LAST_ERR=/tmp/tmp.xggyAjoByc ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.wBUDnmLohy ++ cat /tmp/tmp.xggyAjoByc ++ rm /tmp/tmp.wBUDnmLohy /tmp/tmp.xggyAjoByc ++ return 0 + kubectl_bin delete pod -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-4mfg9 ++ mktemp + local LAST_OUT=/tmp/tmp.93ux87A7qO ++ mktemp + local LAST_ERR=/tmp/tmp.TME7NUKXN5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pod -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-4mfg9 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.93ux87A7qO pod "percona-xtradb-cluster-operator-75d57fb4f6-4mfg9" deleted + cat /tmp/tmp.TME7NUKXN5 + rm /tmp/tmp.93ux87A7qO /tmp/tmp.TME7NUKXN5 + return 0 + kubectl_bin delete pxc --all ++ mktemp + local LAST_OUT=/tmp/tmp.cN0X9C4uMT ++ mktemp + local LAST_ERR=/tmp/tmp.yRZD6oUzKx + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.cN0X9C4uMT perconaxtradbcluster.pxc.percona.com "minimal-cluster" deleted + cat /tmp/tmp.yRZD6oUzKx + rm /tmp/tmp.cN0X9C4uMT /tmp/tmp.yRZD6oUzKx + return 0 + kubectl_bin delete deploy pxc-client ++ mktemp + local LAST_OUT=/tmp/tmp.duv8dUKLqn ++ mktemp + local LAST_ERR=/tmp/tmp.sFXQmCSGfC + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deploy pxc-client + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.duv8dUKLqn deployment.apps "pxc-client" deleted + cat /tmp/tmp.sFXQmCSGfC + rm /tmp/tmp.duv8dUKLqn /tmp/tmp.sFXQmCSGfC + return 0 + sleep 30 + kubectl_bin delete deployment version-service-cr ++ mktemp + local LAST_OUT=/tmp/tmp.aUlpLHsvWB ++ mktemp + local LAST_ERR=/tmp/tmp.pe0sA6B1Y6 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete deployment version-service-cr + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.aUlpLHsvWB deployment.apps "version-service-cr" deleted + cat /tmp/tmp.pe0sA6B1Y6 + rm /tmp/tmp.aUlpLHsvWB /tmp/tmp.pe0sA6B1Y6 + return 0 + desc 'Telemetry testing finished' + set +o xtrace ----------------------------------------------------------------------------------- Telemetry testing finished ----------------------------------------------------------------------------------- + desc 'PXC cluster with version service offline' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster with version service offline ----------------------------------------------------------------------------------- + cp -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/smart-update-version-service-unreachable.yml /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + yq -i eval '.spec.initContainer.image = "perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c"' /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + spinup_pxc smart-update /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5FDgTAt9EK ++ mktemp + local LAST_ERR=/tmp/tmp.Gj1isYpaxo + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5FDgTAt9EK secret/my-cluster-secrets created secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.Gj1isYpaxo + rm /tmp/tmp.5FDgTAt9EK /tmp/tmp.Gj1isYpaxo + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' ++ mktemp + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ + local LAST_OUT=/tmp/tmp.79mO6ojl4K ++ mktemp + local LAST_ERR=/tmp/tmp.lBL2ObTDwj + local exit_status=0 + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' ++ seq 0 2 + /usr/bin/sed -e 's#apply:.*#apply: Never#' + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.79mO6ojl4K deployment.apps/pxc-client created + cat /tmp/tmp.lBL2ObTDwj + rm /tmp/tmp.79mO6ojl4K /tmp/tmp.lBL2ObTDwj + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + cat /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ ++ mktemp + local LAST_OUT=/tmp/tmp.LibgULCqEN ++ mktemp + local LAST_ERR=/tmp/tmp.OPgl4yk4lA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.LibgULCqEN perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.OPgl4yk4lA + rm /tmp/tmp.LibgULCqEN /tmp/tmp.OPgl4yk4lA + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BczmobaJJF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WgEIhD4uzA +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.BczmobaJJF +++ cat /tmp/tmp.WgEIhD4uzA +++ rm /tmp/tmp.BczmobaJJF /tmp/tmp.WgEIhD4uzA +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.4yFg9pAWtc ++ mktemp + local LAST_ERR=/tmp/tmp.y7W7z9IlKK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.4yFg9pAWtc + cat /tmp/tmp.y7W7z9IlKK error: no matching resources found + rm /tmp/tmp.4yFg9pAWtc /tmp/tmp.y7W7z9IlKK + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace Error from server (NotFound): pods "smart-update-haproxy-0" not found smart-update-haproxy-0......................................Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ echo smart-update-pxc-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-pxc-2 + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4jRZshxY2b +++ mktemp ++ local LAST_ERR=/tmp/tmp.ewpk2COrd4 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4jRZshxY2b ++ cat /tmp/tmp.ewpk2COrd4 ++ rm /tmp/tmp.4jRZshxY2b /tmp/tmp.ewpk2COrd4 ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AQ5OuM9r2Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.QAi7fsY8Zd ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.AQ5OuM9r2Y ++ cat /tmp/tmp.QAi7fsY8Zd ++ rm /tmp/tmp.AQ5OuM9r2Y /tmp/tmp.QAi7fsY8Zd ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IPluOseaez +++ mktemp ++ local LAST_ERR=/tmp/tmp.s53CtO9KdG ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IPluOseaez ++ cat /tmp/tmp.s53CtO9KdG ++ rm /tmp/tmp.IPluOseaez /tmp/tmp.s53CtO9KdG ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4Hq3mdkCV1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.lPbDCHo7CH ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.4Hq3mdkCV1 ++ cat /tmp/tmp.lPbDCHo7CH ++ rm /tmp/tmp.4Hq3mdkCV1 /tmp/tmp.lPbDCHo7CH ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JdgWQOAFZK +++ mktemp ++ local LAST_ERR=/tmp/tmp.pYk3mOYhGn ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.JdgWQOAFZK ++ cat /tmp/tmp.pYk3mOYhGn ++ rm /tmp/tmp.JdgWQOAFZK /tmp/tmp.pYk3mOYhGn ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uUSGn76wHi +++ mktemp ++ local LAST_ERR=/tmp/tmp.z55pBJG9D1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uUSGn76wHi ++ cat /tmp/tmp.z55pBJG9D1 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.uUSGn76wHi /tmp/tmp.z55pBJG9D1 ++ return 0 + '[' '' ']' + wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tfbqFkClYw +++ mktemp ++ local LAST_ERR=/tmp/tmp.jSfjU2MqGh ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tfbqFkClYw ++ cat /tmp/tmp.jSfjU2MqGh ++ rm /tmp/tmp.tfbqFkClYw /tmp/tmp.jSfjU2MqGh ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1X1WUA7Aw7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.epQxJrQNdt ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.1X1WUA7Aw7 ++ cat /tmp/tmp.epQxJrQNdt ++ rm /tmp/tmp.1X1WUA7Aw7 /tmp/tmp.epQxJrQNdt ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.pH7ksiq9Kz ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.mR66OE0xfj +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.pH7ksiq9Kz +++++ cat /tmp/tmp.mR66OE0xfj +++++ rm /tmp/tmp.pH7ksiq9Kz /tmp/tmp.mR66OE0xfj +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tpT7vpvkvP +++ mktemp ++ local LAST_ERR=/tmp/tmp.QtS8kbuIms ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tpT7vpvkvP ++ cat /tmp/tmp.QtS8kbuIms ++ rm /tmp/tmp.tpT7vpvkvP /tmp/tmp.QtS8kbuIms ++ return 0 + [[ 2 == \2 ]] ++ kubectl_bin get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ep18e74j3Q +++ mktemp ++ local LAST_ERR=/tmp/tmp.ELCQBCCgmv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc/smart-update -o 'jsonpath={.spec.pxc.image}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ep18e74j3Q ++ cat /tmp/tmp.ELCQBCCgmv ++ rm /tmp/tmp.ep18e74j3Q /tmp/tmp.ELCQBCCgmv ++ return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 != \p\e\r\c\o\n\a\/\p\e\r\c\o\n\a\-\x\t\r\a\d\b\-\c\l\u\s\t\e\r\:\8\.\0\.\1\9\-\1\0\.\1 ]] + desc 'PXC cluster update with recommended image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with recommended image by version service ----------------------------------------------------------------------------------- + vs_image=recommended ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Car29ligcp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.H9fOi6QsVi +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Car29ligcp +++ cat /tmp/tmp.H9fOi6QsVi +++ rm /tmp/tmp.Car29ligcp /tmp/tmp.H9fOi6QsVi +++ return 0 ++ client_pod=pxc-client-6644d8898f-dk4vq ++ wait_pod pxc-client-6644d8898f-dk4vq ++ local pod=pxc-client-6644d8898f-dk4vq ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-dk4vq +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.g15TyAdBxM ++ mktemp + local LAST_ERR=/tmp/tmp.Hu7HuuYn75 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"recommended","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.g15TyAdBxM perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.Hu7HuuYn75 + rm /tmp/tmp.g15TyAdBxM /tmp/tmp.Hu7HuuYn75 + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kJJS0mgHrx +++ mktemp ++ local LAST_ERR=/tmp/tmp.gVKVn5ZFEP ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kJJS0mgHrx ++ cat /tmp/tmp.gVKVn5ZFEP ++ rm /tmp/tmp.kJJS0mgHrx /tmp/tmp.gVKVn5ZFEP ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZhTDSLpCpi +++ mktemp ++ local LAST_ERR=/tmp/tmp.RNHWzpbvkw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.ZhTDSLpCpi ++ cat /tmp/tmp.RNHWzpbvkw ++ rm /tmp/tmp.ZhTDSLpCpi /tmp/tmp.RNHWzpbvkw ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.RGAl4sW2Ja ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.iQNL6GP1fu +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.RGAl4sW2Ja +++++ cat /tmp/tmp.iQNL6GP1fu +++++ rm /tmp/tmp.RGAl4sW2Ja /tmp/tmp.iQNL6GP1fu +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zmj4UIEEBb +++ mktemp ++ local LAST_ERR=/tmp/tmp.ePRVywVYQ5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zmj4UIEEBb ++ cat /tmp/tmp.ePRVywVYQ5 ++ rm /tmp/tmp.zmj4UIEEBb /tmp/tmp.ePRVywVYQ5 ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IQ9E3w0SQJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.55qqHkmUqR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.IQ9E3w0SQJ ++ cat /tmp/tmp.55qqHkmUqR ++ rm /tmp/tmp.IQ9E3w0SQJ /tmp/tmp.55qqHkmUqR ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dl5WE4q7sw +++ mktemp ++ local LAST_ERR=/tmp/tmp.u1pUrmYeRg ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.dl5WE4q7sw ++ cat /tmp/tmp.u1pUrmYeRg ++ rm /tmp/tmp.dl5WE4q7sw /tmp/tmp.u1pUrmYeRg ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SVGgvO68Hq +++ mktemp ++ local LAST_ERR=/tmp/tmp.iLk40hnMv5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.SVGgvO68Hq ++ cat /tmp/tmp.iLk40hnMv5 ++ rm /tmp/tmp.SVGgvO68Hq /tmp/tmp.iLk40hnMv5 ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + kubectl_bin delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.5c5nfBUn9h ++ mktemp + local LAST_ERR=/tmp/tmp.NdM4doGjRL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.5c5nfBUn9h perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.NdM4doGjRL + rm /tmp/tmp.5c5nfBUn9h /tmp/tmp.NdM4doGjRL + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.Rcqib0mRog ++ mktemp + local LAST_ERR=/tmp/tmp.vbKbUBi0G3 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.Rcqib0mRog persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.vbKbUBi0G3 + rm /tmp/tmp.Rcqib0mRog /tmp/tmp.vbKbUBi0G3 + return 0 + desc 'PXC cluster update with the latest image by version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with the latest image by version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.fFbgFnp0M3 ++ mktemp + local LAST_ERR=/tmp/tmp.B4xoDCMi2l + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.fFbgFnp0M3 secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.B4xoDCMi2l + rm /tmp/tmp.fFbgFnp0M3 /tmp/tmp.B4xoDCMi2l + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + kubectl_bin apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' ++ mktemp + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.i6WxICkSZx + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' ++ mktemp + local LAST_ERR=/tmp/tmp.AjbpLfzlGF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.i6WxICkSZx deployment.apps/pxc-client unchanged + cat /tmp/tmp.AjbpLfzlGF + rm /tmp/tmp.i6WxICkSZx /tmp/tmp.AjbpLfzlGF + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + '[' -z '' ']' + kubectl_bin apply -f - + cat_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml ++ mktemp + cat /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + local LAST_OUT=/tmp/tmp.JpbsGNnF2O + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' ++ mktemp + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + local LAST_ERR=/tmp/tmp.NlULKIHS2T + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.JpbsGNnF2O perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.NlULKIHS2T + rm /tmp/tmp.JpbsGNnF2O /tmp/tmp.NlULKIHS2T + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NYX7aDNiEg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xBpkV0oXsR +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.NYX7aDNiEg +++ cat /tmp/tmp.xBpkV0oXsR +++ rm /tmp/tmp.NYX7aDNiEg /tmp/tmp.xBpkV0oXsR +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.U72Q0fpdrL ++ mktemp + local LAST_ERR=/tmp/tmp.tvOTmtbQhN + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.U72Q0fpdrL + cat /tmp/tmp.tvOTmtbQhN error: no matching resources found + rm /tmp/tmp.U72Q0fpdrL /tmp/tmp.tvOTmtbQhN + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ echo smart-update-haproxy-0 ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-0 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-2 ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yXb6LSHpCy +++ mktemp ++ local LAST_ERR=/tmp/tmp.CFBTaDezzi ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.yXb6LSHpCy ++ cat /tmp/tmp.CFBTaDezzi ++ rm /tmp/tmp.yXb6LSHpCy /tmp/tmp.CFBTaDezzi ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6l69hznkTf +++ mktemp ++ local LAST_ERR=/tmp/tmp.mfuy1DcmI5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.6l69hznkTf ++ cat /tmp/tmp.mfuy1DcmI5 ++ rm /tmp/tmp.6l69hznkTf /tmp/tmp.mfuy1DcmI5 ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-dk4vq + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uLprtdyGdl +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nfw8xgrDkw ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.uLprtdyGdl ++ cat /tmp/tmp.Nfw8xgrDkw ++ rm /tmp/tmp.uLprtdyGdl /tmp/tmp.Nfw8xgrDkw ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CuKUUkrFtO +++ mktemp ++ local LAST_ERR=/tmp/tmp.1cIG9LEcT5 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CuKUUkrFtO ++ cat /tmp/tmp.1cIG9LEcT5 ++ rm /tmp/tmp.CuKUUkrFtO /tmp/tmp.1cIG9LEcT5 ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oeVMOhI9Sz +++ mktemp ++ local LAST_ERR=/tmp/tmp.wTWdaLLwfk ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.oeVMOhI9Sz ++ cat /tmp/tmp.wTWdaLLwfk ++ rm /tmp/tmp.oeVMOhI9Sz /tmp/tmp.wTWdaLLwfk ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DHBYaHEMlP +++ mktemp ++ local LAST_ERR=/tmp/tmp.7VixtXk0V1 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.DHBYaHEMlP ++ cat /tmp/tmp.7VixtXk0V1 Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.DHBYaHEMlP /tmp/tmp.7VixtXk0V1 ++ return 0 + '[' '' ']' + vs_image=latest ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Nk6xTVRcOk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aJFJc4vUMp +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.Nk6xTVRcOk +++ cat /tmp/tmp.aJFJc4vUMp +++ rm /tmp/tmp.Nk6xTVRcOk /tmp/tmp.aJFJc4vUMp +++ return 0 ++ client_pod=pxc-client-6644d8898f-dk4vq ++ wait_pod pxc-client-6644d8898f-dk4vq ++ local pod=pxc-client-6644d8898f-dk4vq ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-dk4vq +++ egrep '^(pxc|proxysql)$' +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.L7JGfQ4pm3 ++ mktemp + local LAST_ERR=/tmp/tmp.wMRezeJDDL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"latest","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L7JGfQ4pm3 perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.wMRezeJDDL + rm /tmp/tmp.L7JGfQ4pm3 /tmp/tmp.wMRezeJDDL + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=perconalab/percona-xtradb-cluster-operator:main-pxc8.0 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.BLGkxpjtqG +++ mktemp ++ local LAST_ERR=/tmp/tmp.G6Heua5tvl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.BLGkxpjtqG ++ cat /tmp/tmp.G6Heua5tvl ++ rm /tmp/tmp.BLGkxpjtqG /tmp/tmp.G6Heua5tvl ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.639HiFS79Y +++ mktemp ++ local LAST_ERR=/tmp/tmp.c2t7NtcDGm ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.639HiFS79Y ++ cat /tmp/tmp.c2t7NtcDGm ++ rm /tmp/tmp.639HiFS79Y /tmp/tmp.c2t7NtcDGm ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.vCpAJpcu06 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.YyBziag9ex +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.vCpAJpcu06 +++++ cat /tmp/tmp.YyBziag9ex +++++ rm /tmp/tmp.vCpAJpcu06 /tmp/tmp.YyBziag9ex +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qqwKYuNDqJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.qLPYXiCiBO ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qqwKYuNDqJ ++ cat /tmp/tmp.qLPYXiCiBO ++ rm /tmp/tmp.qqwKYuNDqJ /tmp/tmp.qLPYXiCiBO ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lfJULuFlZh +++ mktemp ++ local LAST_ERR=/tmp/tmp.H0qQp0ys9U ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.lfJULuFlZh ++ cat /tmp/tmp.H0qQp0ys9U ++ rm /tmp/tmp.lfJULuFlZh /tmp/tmp.H0qQp0ys9U ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kMDAFHONWU +++ mktemp ++ local LAST_ERR=/tmp/tmp.7yXW34FvNS ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kMDAFHONWU ++ cat /tmp/tmp.7yXW34FvNS ++ rm /tmp/tmp.kMDAFHONWU /tmp/tmp.7yXW34FvNS ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fHz9OhqUot +++ mktemp ++ local LAST_ERR=/tmp/tmp.dPsiGOtw7w ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.fHz9OhqUot ++ cat /tmp/tmp.dPsiGOtw7w ++ rm /tmp/tmp.fHz9OhqUot /tmp/tmp.dPsiGOtw7w ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + kubectl_bin delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1gV5pJeROg ++ mktemp + local LAST_ERR=/tmp/tmp.MKxz2gfoDm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.1gV5pJeROg perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.MKxz2gfoDm + rm /tmp/tmp.1gV5pJeROg /tmp/tmp.MKxz2gfoDm + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.hrhKf3Qaxe ++ mktemp + local LAST_ERR=/tmp/tmp.FmV9R0TQSy + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.hrhKf3Qaxe persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.FmV9R0TQSy + rm /tmp/tmp.hrhKf3Qaxe /tmp/tmp.FmV9R0TQSy + return 0 + desc 'PXC cluster update with explicitly specified image inside version service' + set +o xtrace ----------------------------------------------------------------------------------- PXC cluster update with explicitly specified image inside version service ----------------------------------------------------------------------------------- + spinup_pxc smart-update /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local cluster=smart-update + local config=/tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + local size=3 + local sleep=10 + local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + local port=3306 + desc 'create first PXC cluster' + set +o xtrace ----------------------------------------------------------------------------------- create first PXC cluster ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml ++ mktemp + local LAST_OUT=/tmp/tmp.adY4UK0jbA ++ mktemp + local LAST_ERR=/tmp/tmp.JnBMhOtUbt + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/secrets.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.adY4UK0jbA secret/my-cluster-secrets unchanged secret/some-name-ssl unchanged secret/some-name-ssl-internal unchanged + cat /tmp/tmp.JnBMhOtUbt + rm /tmp/tmp.adY4UK0jbA /tmp/tmp.JnBMhOtUbt + return 0 + apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + '[' -z '' ']' + cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + kubectl_bin apply -f - + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' ++ mktemp + local LAST_OUT=/tmp/tmp.YzdNJvrgqZ + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + local LAST_ERR=/tmp/tmp.mWtzKcQpzj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/conf/client.yml + /usr/bin/sed -e 's#apply:.*#apply: Never#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.YzdNJvrgqZ deployment.apps/pxc-client unchanged + cat /tmp/tmp.mWtzKcQpzj + rm /tmp/tmp.YzdNJvrgqZ /tmp/tmp.mWtzKcQpzj + return 0 + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + apply_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + '[' -z '' ']' + cat_config /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#' + /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#' + /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1714-41f82f9c#' + cat /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#' + /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#' + /usr/bin/sed -e s~minio-service.#namespace~minio-service.smart-update2-20475~ + /usr/bin/sed -e 's#apply:.*#apply: Never#' + kubectl_bin apply -f - + /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#' + /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v9-9-9#' + /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' + /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: percona/percona-xtradb-cluster:8.0.19-10.1#' ++ mktemp + local LAST_OUT=/tmp/tmp.rbeOEj52o0 ++ mktemp + local LAST_ERR=/tmp/tmp.sO32yGNobX + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl apply -f - + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.rbeOEj52o0 perconaxtradbcluster.pxc.percona.com/smart-update created + cat /tmp/tmp.sO32yGNobX + rm /tmp/tmp.rbeOEj52o0 /tmp/tmp.sO32yGNobX + return 0 + desc 'check if all 3 Pods started' + set +o xtrace ----------------------------------------------------------------------------------- check if all 3 Pods started ----------------------------------------------------------------------------------- ++ get_proxy smart-update ++ local target_cluster=smart-update +++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.g7F0xIYpSt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uvHVT7bmeD +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.g7F0xIYpSt +++ cat /tmp/tmp.uvHVT7bmeD +++ rm /tmp/tmp.g7F0xIYpSt /tmp/tmp.uvHVT7bmeD +++ return 0 ++ [[ true == \t\r\u\e ]] ++ echo smart-update-haproxy ++ return + local proxy=smart-update-haproxy + kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 ++ mktemp + local LAST_OUT=/tmp/tmp.QnhL2ojXyG ++ mktemp + local LAST_ERR=/tmp/tmp.zsf4bw8SLr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + for i in '$(seq 0 2)' + set +e + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n smart-update2-20475 + exit_status=1 + set -e + '[' 1 '!=' 0 ']' + '[' 1 == 1 ']' + sleep 0 + cat /tmp/tmp.QnhL2ojXyG + cat /tmp/tmp.zsf4bw8SLr error: no matching resources found + rm /tmp/tmp.QnhL2ojXyG /tmp/tmp.zsf4bw8SLr + return 1 + true + wait_for_running smart-update-haproxy 1 + local name=smart-update-haproxy + let last_pod=0 + : + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 0 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-haproxy-0 480 + local pod=smart-update-haproxy-0 + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-haproxy-0 + local container= + set +o xtrace pod/smart-update-haproxy-0 condition met smart-update-haproxy-0Defaulted container "haproxy" out of: haproxy, pxc-monit, pxc-init (init) .Ok + wait_for_running smart-update-pxc 3 + local name=smart-update-pxc + let last_pod=2 + local max_retry=480 + desc 'wait for running cluster' + set +o xtrace ----------------------------------------------------------------------------------- wait for running cluster ----------------------------------------------------------------------------------- ++ seq 0 2 + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-0 480 + local pod=smart-update-pxc-0 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo smart-update-pxc-0 + local container=pxc + set +o xtrace pod/smart-update-pxc-0 condition met smart-update-pxc-0.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-1 480 + local pod=smart-update-pxc-1 + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ echo smart-update-pxc-1 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container=pxc + set +o xtrace pod/smart-update-pxc-1 condition met smart-update-pxc-1.Ok + for i in '$(seq 0 $last_pod)' + wait_pod smart-update-pxc-2 480 + local pod=smart-update-pxc-2 + local max_retry=480 + local ns= ++ echo smart-update-pxc-2 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container=pxc + set +o xtrace pod/smart-update-pxc-2 condition met smart-update-pxc-2.Ok + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 5\.7 ]] + run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qPuzlN65Gl +++ mktemp ++ local LAST_ERR=/tmp/tmp.8oBU35cbkZ ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.qPuzlN65Gl ++ cat /tmp/tmp.8oBU35cbkZ ++ rm /tmp/tmp.qPuzlN65Gl /tmp/tmp.8oBU35cbkZ ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-dk4vq + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h smart-update-haproxy -uroot -proot_password -P3306' + local 'command=INSERT myApp.myApp (id) VALUES (100500)' + local 'uri=-h smart-update-haproxy -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.pFhJoudTso +++ mktemp ++ local LAST_ERR=/tmp/tmp.rFOuPNnnD0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pFhJoudTso ++ cat /tmp/tmp.rFOuPNnnD0 ++ rm /tmp/tmp.pFhJoudTso /tmp/tmp.rFOuPNnnD0 ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + sleep 30 ++ seq 0 2 + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f8OaJuICHP +++ mktemp ++ local LAST_ERR=/tmp/tmp.5z8S6eZz2f ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.f8OaJuICHP ++ cat /tmp/tmp.5z8S6eZz2f ++ rm /tmp/tmp.f8OaJuICHP /tmp/tmp.5z8S6eZz2f ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.urErHAUEJv +++ mktemp ++ local LAST_ERR=/tmp/tmp.Kb0qpyNhJM ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.urErHAUEJv ++ cat /tmp/tmp.Kb0qpyNhJM ++ rm /tmp/tmp.urErHAUEJv /tmp/tmp.Kb0qpyNhJM ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((size - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password -P3306' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HMeyFgGQtA +++ mktemp ++ local LAST_ERR=/tmp/tmp.zXDVyK7DVl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.HMeyFgGQtA ++ cat /tmp/tmp.zXDVyK7DVl ++ rm /tmp/tmp.HMeyFgGQtA /tmp/tmp.zXDVyK7DVl ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ echo pxc-client-6644d8898f-dk4vq + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql ++ is_keyring_plugin_in_use smart-update ++ local cluster=smart-update ++ kubectl_bin exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ egrep -o 'early-plugin-load=keyring_\w+.so' +++ mktemp ++ local LAST_OUT=/tmp/tmp.j5F3x4i6J6 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Nbr38m5HMx ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -it smart-update-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.j5F3x4i6J6 ++ cat /tmp/tmp.Nbr38m5HMx Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.j5F3x4i6J6 /tmp/tmp.Nbr38m5HMx ++ return 0 + '[' '' ']' ++ sort -V ++ head -n1 +++ get_operator_pod ++ grep :8.0 +++ local label_prefix=app.kubernetes.io/ ++++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator ++++ grep -c percona-xtradb-cluster-operator ++ jq -r '.versions[].matrix.pxc[].imagePath' ++ tail -n2 +++ local check_label=1 +++ [[ 1 -eq 0 ]] +++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qK6BXQz7IH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.686InUrbsN +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.qK6BXQz7IH +++ cat /tmp/tmp.686InUrbsN +++ rm /tmp/tmp.qK6BXQz7IH /tmp/tmp.686InUrbsN +++ return 0 ++ kubectl_bin exec -ti percona-xtradb-cluster-operator-75d57fb4f6-hfdc6 -n pxc-operator -- curl -s http://version-service.smart-update2-20475.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 +++ mktemp ++ local LAST_OUT=/tmp/tmp.J9FUDlcIq5 +++ mktemp ++ local LAST_ERR=/tmp/tmp.9I9TvNc7yv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl exec -ti percona-xtradb-cluster-operator-75d57fb4f6-hfdc6 -n pxc-operator -- curl -s http://version-service.smart-update2-20475.svc.cluster.local:11000/versions/v1/pxc-operator/9.9.9 ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.J9FUDlcIq5 ++ cat /tmp/tmp.9I9TvNc7yv Unable to use a TTY - input is not a terminal or the right kind of file ++ rm /tmp/tmp.J9FUDlcIq5 /tmp/tmp.9I9TvNc7yv ++ return 0 + vs_image=percona/percona-xtradb-cluster:8.0.19-10.1 ++ run_mysql 'SELECT @@hostname hostname;' '-h smart-update-haproxy -uroot -proot_password' ++ local 'command=SELECT @@hostname hostname;' ++ local 'uri=-h smart-update-haproxy -uroot -proot_password' +++ get_client_pod +++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CKZLeiQi0r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ca7HZNXiIe +++ local exit_status=0 ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ set +e +++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ set -e +++ '[' 0 '!=' 0 ']' +++ break +++ cat /tmp/tmp.CKZLeiQi0r +++ cat /tmp/tmp.ca7HZNXiIe +++ rm /tmp/tmp.CKZLeiQi0r /tmp/tmp.ca7HZNXiIe +++ return 0 ++ client_pod=pxc-client-6644d8898f-dk4vq ++ wait_pod pxc-client-6644d8898f-dk4vq ++ local pod=pxc-client-6644d8898f-dk4vq ++ local max_retry=480 ++ local ns= +++ echo pxc-client-6644d8898f-dk4vq +++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' +++ egrep '^(pxc|proxysql)$' ++ local container= ++ set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok ++ set +o xtrace + initial_primary=smart-update-pxc-0 + kubectl_bin patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' ++ mktemp + local LAST_OUT=/tmp/tmp.L8HiDySnfW ++ mktemp + local LAST_ERR=/tmp/tmp.BtCigqroHf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl patch pxc/smart-update --type=merge -p '{"spec":{"upgradeOptions":{"versionServiceEndpoint":"http://version-service:11000","apply":"percona/percona-xtradb-cluster:8.0.19-10.1","schedule": "* * * * *"}}}' + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.L8HiDySnfW perconaxtradbcluster.pxc.percona.com/smart-update patched + cat /tmp/tmp.BtCigqroHf + rm /tmp/tmp.L8HiDySnfW /tmp/tmp.BtCigqroHf + return 0 + sleep 55 + check_last_pod_to_update smart-update smart-update-pxc-0 3 percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + local cluster=smart-update + local initial_primary=smart-update-pxc-0 + local pxc_size=3 + local target_image=percona/percona-xtradb-cluster:percona/percona-xtradb-cluster:8.0.19-10.1 + set +x Waiting for the last pod to update+ wait_cluster_consistency smart-update 3 2 + local cluster_name=smart-update + local cluster_size=3 + local proxy_size=2 + '[' -z 2 ']' + desc 'wait cluster consistency' + set +o xtrace ----------------------------------------------------------------------------------- wait cluster consistency ----------------------------------------------------------------------------------- + local i=0 + local max=36 + sleep 7 ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.state}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rKQwmDPOyJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.YzVkLtxGlr ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.state}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.rKQwmDPOyJ ++ cat /tmp/tmp.YzVkLtxGlr ++ rm /tmp/tmp.rKQwmDPOyJ /tmp/tmp.YzVkLtxGlr ++ return 0 + [[ ready == \r\e\a\d\y ]] ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.pxc.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.F8CJT43VPw +++ mktemp ++ local LAST_ERR=/tmp/tmp.PVQlHo4SCl ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.pxc.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.F8CJT43VPw ++ cat /tmp/tmp.PVQlHo4SCl ++ rm /tmp/tmp.F8CJT43VPw /tmp/tmp.PVQlHo4SCl ++ return 0 + [[ 3 == \3 ]] +++ get_proxy_engine smart-update +++ local cluster_name=smart-update ++++ get_proxy smart-update ++++ local target_cluster=smart-update +++++ kubectl_bin get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.19UjJBz2K1 ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.PhJ7SOTx9X +++++ local exit_status=0 ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ set +e +++++ kubectl get pxc smart-update -o 'jsonpath={.spec.haproxy.enabled}' +++++ exit_status=0 +++++ set -e +++++ '[' 0 '!=' 0 ']' +++++ break +++++ cat /tmp/tmp.19UjJBz2K1 +++++ cat /tmp/tmp.PhJ7SOTx9X +++++ rm /tmp/tmp.19UjJBz2K1 /tmp/tmp.PhJ7SOTx9X +++++ return 0 ++++ [[ true == \t\r\u\e ]] ++++ echo smart-update-haproxy ++++ return +++ local cluster_proxy=smart-update-haproxy +++ echo haproxy ++ kubectl_bin get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tbjFjrCSOb +++ mktemp ++ local LAST_ERR=/tmp/tmp.zhUASAIupF ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pxc smart-update -o 'jsonpath={.status.haproxy.ready}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.tbjFjrCSOb ++ cat /tmp/tmp.zhUASAIupF ++ rm /tmp/tmp.tbjFjrCSOb /tmp/tmp.zhUASAIupF ++ return 0 + [[ 2 == \2 ]] ++ seq 0 2 + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-0.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CwwI0gQhEu +++ mktemp ++ local LAST_ERR=/tmp/tmp.1H1uniPPQq ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.CwwI0gQhEu ++ cat /tmp/tmp.1H1uniPPQq ++ rm /tmp/tmp.CwwI0gQhEu /tmp/tmp.1H1uniPPQq ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' ++ echo pxc-client-6644d8898f-dk4vq + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-1.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.kbi3jhvwMD +++ mktemp ++ local LAST_ERR=/tmp/tmp.H5oS0XPP4g ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.kbi3jhvwMD ++ cat /tmp/tmp.H5oS0XPP4g ++ rm /tmp/tmp.kbi3jhvwMD /tmp/tmp.H5oS0XPP4g ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + for i in '$(seq 0 $((CLUSTER_SIZE - 1)))' + compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local command_id=select-1 + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql + [[ percona/percona-xtradb-cluster:8.0.19-10.1 =~ 8\.0 ]] + '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1-80.sql ']' + run_mysql 'SELECT * from myApp.myApp;' '-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' + local 'command=SELECT * from myApp.myApp;' + local 'uri=-h smart-update-pxc-2.smart-update-pxc -uroot -proot_password' ++ get_client_pod ++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zWdpjRBTPj +++ mktemp ++ local LAST_ERR=/tmp/tmp.ShNvrU2WJy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.zWdpjRBTPj ++ cat /tmp/tmp.ShNvrU2WJy ++ rm /tmp/tmp.zWdpjRBTPj /tmp/tmp.ShNvrU2WJy ++ return 0 + client_pod=pxc-client-6644d8898f-dk4vq + wait_pod pxc-client-6644d8898f-dk4vq + local pod=pxc-client-6644d8898f-dk4vq + local max_retry=480 + local ns= ++ echo pxc-client-6644d8898f-dk4vq ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace pod/pxc-client-6644d8898f-dk4vq condition met pxc-client-6644d8898f-dk4vq.Ok + set +o xtrace + '[' '!' -s /tmp/tmp.qA2QIzYgTk/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/compare/select-1.sql /tmp/tmp.qA2QIzYgTk/select-1.sql + kubectl_bin delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml ++ mktemp + local LAST_OUT=/tmp/tmp.37uqpqkIf8 ++ mktemp + local LAST_ERR=/tmp/tmp.734cxg9V6R + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /tmp/tmp.qA2QIzYgTk/smart-update-version-service-unreachable.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.37uqpqkIf8 perconaxtradbcluster.pxc.percona.com "smart-update" deleted + cat /tmp/tmp.734cxg9V6R + rm /tmp/tmp.37uqpqkIf8 /tmp/tmp.734cxg9V6R + return 0 + kubectl_bin delete pvc --all ++ mktemp + local LAST_OUT=/tmp/tmp.NH6o4rPg25 ++ mktemp + local LAST_ERR=/tmp/tmp.iuKfIUBqOg + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pvc --all + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.NH6o4rPg25 persistentvolumeclaim "datadir-smart-update-pxc-0" deleted persistentvolumeclaim "datadir-smart-update-pxc-1" deleted persistentvolumeclaim "datadir-smart-update-pxc-2" deleted + cat /tmp/tmp.iuKfIUBqOg + rm /tmp/tmp.NH6o4rPg25 /tmp/tmp.iuKfIUBqOg + return 0 + desc cleanup + set +o xtrace ----------------------------------------------------------------------------------- cleanup ----------------------------------------------------------------------------------- + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml ++ mktemp + local LAST_OUT=/tmp/tmp.xAH2YJxxL8 ++ mktemp + local LAST_ERR=/tmp/tmp.bcF9MYO8DL + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1714/e2e-tests/smart-update2/conf/vs.yml + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.xAH2YJxxL8 deployment.apps "version-service" deleted service "version-service" deleted + cat /tmp/tmp.bcF9MYO8DL + rm /tmp/tmp.xAH2YJxxL8 /tmp/tmp.bcF9MYO8DL + return 0 + destroy smart-update2-20475 + local namespace=smart-update2-20475 + local ignore_logs=true + desc 'destroy cluster/operator and all other resources' + set +o xtrace ----------------------------------------------------------------------------------- destroy cluster/operator and all other resources ----------------------------------------------------------------------------------- + '[' true == false -o 1 == 1 ']' + grep -v 'get backup status: Job.batch' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + tee /tmp/tmp.qA2QIzYgTk/operator.log + sort -u + grep -v 'the object has been modified' + grep -v level=info ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.pvfk3lvIRu +++ mktemp ++ local LAST_ERR=/tmp/tmp.78vVVz7ysv ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ set +e ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ set -e ++ '[' 0 '!=' 0 ']' ++ break ++ cat /tmp/tmp.pvfk3lvIRu ++ cat /tmp/tmp.78vVVz7ysv ++ rm /tmp/tmp.pvfk3lvIRu /tmp/tmp.78vVVz7ysv ++ return 0 + kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-hfdc6 ++ mktemp + local LAST_OUT=/tmp/tmp.EZEd3IQFbJ ++ mktemp + local LAST_ERR=/tmp/tmp.GylnxJh69Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl logs -n pxc-operator percona-xtradb-cluster-operator-75d57fb4f6-hfdc6 + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.EZEd3IQFbJ + cat /tmp/tmp.GylnxJh69Q + rm /tmp/tmp.EZEd3IQFbJ /tmp/tmp.GylnxJh69Q + return 0 2024-05-16T03:03:35.060Z INFO setup Runs on {"platform": "kubernetes", "version": "v1.26.15-gke.1300000"} 2024-05-16T03:03:35.061Z INFO setup Manager starting up {"gitCommit": "41f82f9c8696cc17525c3cbb8d044dbe4590ed84", "gitBranch": "PR-1714-41f82f9c", "buildTime": "2024-05-16T01:35:13Z", "goVersion": "go1.22.3", "os": "linux", "arch": "amd64"} 2024-05-16T03:03:35.062Z INFO setup Registering Components. 2024-05-16T03:03:38.310Z INFO controller-runtime.webhook Registering webhook {"path": "/validate-percona-xtradbcluster"} 2024-05-16T03:03:38.313Z INFO controller-runtime.metrics Serving metrics server {"bindAddress": ":8080", "secure": false} 2024-05-16T03:03:38.313Z INFO controller-runtime.metrics Starting metrics server 2024-05-16T03:03:38.313Z INFO controller-runtime.webhook Starting webhook server 2024-05-16T03:03:38.313Z INFO setup Starting the Cmd. 2024-05-16T03:03:38.313Z INFO starting server {"name": "health probe", "addr": "[::]:8081"} 2024-05-16T03:03:38.314Z INFO controller-runtime.certwatcher Starting certificate watcher 2024-05-16T03:03:38.314Z INFO controller-runtime.certwatcher Updated current TLS certificate 2024-05-16T03:03:38.314Z INFO controller-runtime.webhook Serving webhook server {"host": "", "port": 9443} 2024-05-16T03:03:38.415Z INFO attempting to acquire leader lease pxc-operator/08db1feb.percona.com... 2024-05-16T03:03:54.578Z DEBUG events percona-xtradb-cluster-operator-75d57fb4f6-hfdc6_5d8d2b94-08e8-43e4-a47d-5bd95a182949 became leader {"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"3b814fe2-88f9-4091-bc36-ff612bc18b7b","apiVersion":"coordination.k8s.io/v1","resourceVersion":"46866"}, "reason": "LeaderElection"} 2024-05-16T03:03:54.578Z INFO successfully acquired lease pxc-operator/08db1feb.percona.com 2024-05-16T03:03:54.579Z INFO Starting Controller {"controller": "pxcbackup-controller"} 2024-05-16T03:03:54.579Z INFO Starting Controller {"controller": "pxc-controller"} 2024-05-16T03:03:54.579Z INFO Starting Controller {"controller": "pxcrestore-controller"} 2024-05-16T03:03:54.579Z INFO Starting EventSource {"controller": "pxcbackup-controller", "source": "kind source: unknown type"} 2024-05-16T03:03:54.579Z INFO Starting EventSource {"controller": "pxc-controller", "source": "kind source: unknown type"} 2024-05-16T03:03:54.579Z INFO Starting EventSource {"controller": "pxcrestore-controller", "source": "kind source: unknown type"} 2024-05-16T03:03:54.792Z INFO Starting workers {"controller": "pxc-controller", "worker count": 1} 2024-05-16T03:03:54.792Z INFO Starting workers {"controller": "pxcrestore-controller", "worker count": 1} 2024-05-16T03:03:54.793Z INFO Starting workers {"controller": "pxcbackup-controller", "worker count": 1} 2024-05-16T03:04:34.043Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "83fb72a5-ebbf-47d5-b25f-e9b8e1265605", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:40.977Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "54a19794-b490-4360-8b71-4f4faaeb65a9", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:41.160Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "54a19794-b490-4360-8b71-4f4faaeb65a9"} 2024-05-16T03:04:41.380Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 49c9aeef-bc04-489e-8b7d-4211d21197c5 2024-05-16T03:04:41.465Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3914659f-580b-4910-bc68-ec200beb9ab9", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:41.577Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3914659f-580b-4910-bc68-ec200beb9ab9"} 2024-05-16T03:04:46.466Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3bc437c5-f639-4d6b-bbf2-b6af779059ff", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:46.563Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3bc437c5-f639-4d6b-bbf2-b6af779059ff"} 2024-05-16T03:04:51.597Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0ad5c3da-de45-41f4-b89f-6936127a3b14", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:51.676Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0ad5c3da-de45-41f4-b89f-6936127a3b14"} 2024-05-16T03:04:56.749Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "72d3a5ed-4c17-4c02-bd23-aaeee5fc9e6c", "oldSize": 1, "newSize": 2} 2024-05-16T03:04:56.824Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "72d3a5ed-4c17-4c02-bd23-aaeee5fc9e6c"} 2024-05-16T03:05:01.866Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a6b11d6d-36e8-480a-b7a5-8b11614d0f25", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:02.184Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a6b11d6d-36e8-480a-b7a5-8b11614d0f25"} 2024-05-16T03:05:07.243Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "4aadf089-666d-4c87-8691-26cbf820f253", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:07.344Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "4aadf089-666d-4c87-8691-26cbf820f253"} 2024-05-16T03:05:12.378Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3589ae77-919b-4b08-a03d-1f4d88b7da55", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:12.480Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3589ae77-919b-4b08-a03d-1f4d88b7da55"} 2024-05-16T03:05:17.570Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2837678d-e34b-4d51-aee9-f2fd32021d11", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:17.675Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2837678d-e34b-4d51-aee9-f2fd32021d11"} 2024-05-16T03:05:22.726Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7dc00a74-b4a5-405b-8f72-0ff1757d8598", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:22.873Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7dc00a74-b4a5-405b-8f72-0ff1757d8598"} 2024-05-16T03:05:27.961Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e69bf181-9657-4a55-aa38-9691d3821a63", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:28.034Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e69bf181-9657-4a55-aa38-9691d3821a63"} 2024-05-16T03:05:33.071Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "81b44d99-c692-4e32-86c0-fca08cd459b2", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:33.144Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "81b44d99-c692-4e32-86c0-fca08cd459b2"} 2024-05-16T03:05:38.182Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "45d78dfd-5fd7-4c7f-a886-f8af85632d33", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:38.281Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "45d78dfd-5fd7-4c7f-a886-f8af85632d33"} 2024-05-16T03:05:43.348Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53a5a780-90dc-43cc-8118-2499a9d66eaf", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:43.471Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53a5a780-90dc-43cc-8118-2499a9d66eaf"} 2024-05-16T03:05:48.518Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2f79011c-014a-4b0f-904e-d68b62982768", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:48.598Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2f79011c-014a-4b0f-904e-d68b62982768"} 2024-05-16T03:05:53.643Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "327ff2ac-0ef2-4507-8781-569b7d8df9e2", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:53.733Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "327ff2ac-0ef2-4507-8781-569b7d8df9e2"} 2024-05-16T03:05:53.824Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:53.915Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "user": "operator"} 2024-05-16T03:05:53.960Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "user": "monitor"} 2024-05-16T03:05:54.045Z INFO User monitor: granted privileges {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed"} 2024-05-16T03:05:54.107Z INFO monitor user privileges granted {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed"} 2024-05-16T03:05:54.146Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "user": "xtrabackup"} 2024-05-16T03:05:54.219Z INFO User xtrabackup: granted privileges {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed"} 2024-05-16T03:05:54.264Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "user": "replication"} 2024-05-16T03:05:54.340Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed"} 2024-05-16T03:05:56.349Z INFO reconcile replication error {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63c5edd2-043d-4603-b9e4-7bba36feb7ed", "err": "get primary pxc pod: failed to get proxy connection: dial tcp 10.154.14.219:3306: connect: connection refused"} 2024-05-16T03:05:58.825Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7e37a5b2-05a2-487d-8765-dc11a8859fab", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:59.018Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7e37a5b2-05a2-487d-8765-dc11a8859fab"} 2024-05-16T03:05:59.124Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0c68ab40-3d0e-4aad-aaae-60feeb0c7293", "oldSize": 1, "newSize": 2} 2024-05-16T03:05:59.257Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0c68ab40-3d0e-4aad-aaae-60feeb0c7293"} 2024-05-16T03:06:04.125Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a2a9fcca-c0b3-4810-891c-2dbbf55e336d", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:04.255Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a2a9fcca-c0b3-4810-891c-2dbbf55e336d"} 2024-05-16T03:06:09.357Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53de3418-f359-4fea-86b8-d60693a49152", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:09.502Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53de3418-f359-4fea-86b8-d60693a49152"} 2024-05-16T03:06:14.583Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e0b25eee-d87e-41e8-9d0d-8597f9608cea", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:14.715Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e0b25eee-d87e-41e8-9d0d-8597f9608cea"} 2024-05-16T03:06:19.795Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d966976b-de63-4623-b32a-85c44e16da05", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:20.375Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d966976b-de63-4623-b32a-85c44e16da05"} 2024-05-16T03:06:20.535Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e305f798-462c-41ab-b1c3-96999f6fd842", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:25.535Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "60743fd5-f06c-4ad2-863c-1b4cd200f1b7", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:31.728Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ab567db7-3c5a-4479-bdf4-6068931f6a01", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:38.178Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7ba31a23-2086-49ca-b285-6dc9f9e99231", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:44.393Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "47136253-b7c0-4985-bfb3-be66792abfd2", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:50.596Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9eda49fd-9539-41bc-8766-4b20e3ff41a3", "oldSize": 1, "newSize": 2} 2024-05-16T03:06:56.828Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ea79b296-b877-4e61-90ea-fc006932f6b1", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:03.028Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "cfe7aed0-ca18-43de-b22d-d54128523b44", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:09.263Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1d111a94-81cd-416c-a801-c6955dbbac6e", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:10.546Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c8785cd2-eaac-4c5a-8adc-9bb1b28c5bb2", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:15.547Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "254aedc0-df02-4775-b890-f01014462ea8", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:22.030Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3411a7c2-bb9c-47c2-974f-a0ea5aaa6aa6", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:28.252Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b18075cd-c619-41b3-8029-795dd76a68f1", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:34.487Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f0a29f25-4ad6-403e-9de4-492db3b10382", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:40.742Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6ea9858c-67ce-47b8-8c13-93e1694f92bf", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:46.941Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2df28a8c-5a60-4852-85b4-6b3b49e06960", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:53.309Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30968769-e2d2-4539-bf57-411f8855a340", "oldSize": 1, "newSize": 2} 2024-05-16T03:07:59.636Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0aa84054-3762-4d95-8275-68720b9cf770", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:05.853Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "28fde6ff-436d-4d95-8e3a-8222a460ac20", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:12.199Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0064ecd4-9818-4b1f-ae36-518cd2e7ddc1", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:18.443Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "028341bf-2eca-4670-85e3-dc86bb7d6c7c", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:19.726Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f9d332c7-98a9-432a-86d9-b8e29a66f773", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:19.901Z INFO Password expiration policy updated {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f9d332c7-98a9-432a-86d9-b8e29a66f773", "user": "root"} 2024-05-16T03:08:21.379Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f9d332c7-98a9-432a-86d9-b8e29a66f773", "new version": "8.0.19-10"} 2024-05-16T03:08:21.509Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b512aac6-e1c7-4775-8dae-b1bcc1b8071b", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:24.726Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "27c5fdcd-19ac-4d14-89b8-ada61b856f58", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:31.031Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b32f6ffc-a10f-459d-9742-028fee660a75", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:37.344Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "016b3dc3-9500-442e-9388-715671f376e4", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:43.604Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1f7c16a7-e64b-4c2c-aee3-b58d07c8363b", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:49.903Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "465050f9-cee6-4752-99e0-d2ae6da82d35", "oldSize": 1, "newSize": 2} 2024-05-16T03:08:56.596Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "63b86938-9ba2-4126-b31f-3170a53008ff", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:02.872Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "784c15b8-4915-4428-8f45-0cdc011383f0", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:09.153Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6fa5c584-23cc-475a-aa75-9872b99113fb", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:15.493Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "be6d13bd-5bbe-4894-88bc-f138bc4334da", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:22.021Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6e12dd00-750e-4d33-b1a2-a1e349405a3a", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:28.319Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "688b4019-b4c7-4e46-ba97-d5d1ad3178fc", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:34.616Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "73a8c1ca-745b-49dc-9174-5d8ab8f4af91", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:41.420Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "4724acd2-be60-4243-848d-f62b1bf0946c", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:47.718Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1a9bc7de-e5fd-4eb5-addd-768330700f95", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:52.150Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:53.420Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "name": "ensure-version/smart-update2-20475/smart-update", "schedule": "* * * * *"} 2024-05-16T03:09:53.495Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9806094d-7ad2-44b9-b707-49d0b89ef850", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:54.782Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "95c52169-dd6f-446a-b542-185498c3fed3", "oldSize": 1, "newSize": 2} 2024-05-16T03:09:59.783Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "184f23b9-c226-4b2b-9854-551efa1c35d0", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:06.053Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "863fe8cc-5242-4aed-b3d4-ad9fd57c82a3", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:12.467Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b8196102-c1ae-461b-8edc-d26724e3b93d", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:18.821Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5d3ffcb3-8fc4-4ce7-9bb5-f49312b83dc6", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:25.743Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ee5eb54b-4924-47d5-8c0f-b8180bd558b5", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:32.000Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b30594ad-f448-49a1-8871-8e63dc6fee85", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:38.402Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "fb766650-8fe0-44b7-8cd0-87d742ce2865", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:44.703Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "952b011f-081e-40cb-ba94-b9c1c2f585db", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:50.974Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8146f566-9695-4fb5-a898-e49512abb535", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:53.000Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "oldSize": 1, "newSize": 2} 2024-05-16T03:10:53.001Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "endpoint": "http://version-service.smart-update2-20475.svc.cluster.local:11000"} 2024-05-16T03:10:53.081Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=49c9aeef-bc04-489e-8b7d-4211d21197c5&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/recommended?clusterWideEnabled=true&customResourceUid=49c9aeef-bc04-489e-8b7d-4211d21197c5&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-05-16T03:10:57.258Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "05ec9b08-09e1-4bdb-a431-ad5b9eae137e", "oldSize": 1, "newSize": 2} 2024-05-16T03:11:03.569Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c602003a-c11f-47f9-9dba-d55c9238f321", "oldSize": 1, "newSize": 2} 2024-05-16T03:11:09.907Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "eba92907-7db7-4e46-a310-a44b61ad116e", "oldSize": 1, "newSize": 2} 2024-05-16T03:11:16.187Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "019ec3d0-81c5-4b80-acf5-e7631c4d4199", "oldSize": 1, "newSize": 2} 2024-05-16T03:11:17.656Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c8e1f547-f86d-4ab7-a424-48765ba6305f", "oldSize": 1, "newSize": 2} 2024-05-16T03:11:56.027Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2091497-2598-4862-8864-9f02ce782785", "name": "ensure-version/smart-update2-20475/smart-update", "cluster": "smart-update", "namespace": "smart-update2-20475"} 2024-05-16T03:12:19.683Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "eecd5a65-dae9-4b8e-89f7-84410202a47f", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:20.318Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "eecd5a65-dae9-4b8e-89f7-84410202a47f"} 2024-05-16T03:12:21.251Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: 38d3bbba-b391-4c8f-981e-ce3ba4e41ab0 2024-05-16T03:12:21.329Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "da6b179c-0aab-4bd9-bc0b-7ed356eb99a4", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:21.431Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "da6b179c-0aab-4bd9-bc0b-7ed356eb99a4"} 2024-05-16T03:12:21.480Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "78bcb0de-b031-4ebc-8764-05be918db6d0", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:21.575Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "78bcb0de-b031-4ebc-8764-05be918db6d0"} 2024-05-16T03:12:26.329Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1a4e1c43-0216-4b17-8a5a-7b8a7d9bd1ad", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:26.479Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1a4e1c43-0216-4b17-8a5a-7b8a7d9bd1ad"} 2024-05-16T03:12:31.534Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6b948eec-a40f-4eef-b4d0-66596c3d689a", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:31.609Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6b948eec-a40f-4eef-b4d0-66596c3d689a"} 2024-05-16T03:12:36.648Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "84926bbe-d859-425e-ab08-e2e8c023b4fd", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:36.725Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "84926bbe-d859-425e-ab08-e2e8c023b4fd"} 2024-05-16T03:12:41.786Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7505a97d-d6b5-4e68-8209-58c47d5e6bb2", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:41.901Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7505a97d-d6b5-4e68-8209-58c47d5e6bb2"} 2024-05-16T03:12:46.937Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b502bee4-b658-4a69-ba76-abdc33e398c8", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:47.241Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b502bee4-b658-4a69-ba76-abdc33e398c8"} 2024-05-16T03:12:52.288Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6187bcb3-0026-4dfa-b47d-790a36f2038e", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:52.398Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6187bcb3-0026-4dfa-b47d-790a36f2038e"} 2024-05-16T03:12:57.435Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8a5815f3-29e9-42fa-8369-4a5b4a1b45a1", "oldSize": 1, "newSize": 2} 2024-05-16T03:12:57.515Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8a5815f3-29e9-42fa-8369-4a5b4a1b45a1"} 2024-05-16T03:13:02.555Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "851f68e7-c986-4962-be57-0232bc8e64eb", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:02.693Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "851f68e7-c986-4962-be57-0232bc8e64eb"} 2024-05-16T03:13:07.751Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5d30bf15-2b74-4075-85d3-33010dce0e84", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:07.840Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5d30bf15-2b74-4075-85d3-33010dce0e84"} 2024-05-16T03:13:12.878Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0c1e8d0c-b735-4512-8e3b-41bde249994f", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:12.972Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0c1e8d0c-b735-4512-8e3b-41bde249994f"} 2024-05-16T03:13:18.015Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2af213fe-1580-4ca4-a8cd-afea3c18dd61", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:18.105Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2af213fe-1580-4ca4-a8cd-afea3c18dd61"} 2024-05-16T03:13:23.138Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e5c75c73-4260-4eb1-8fb7-2a786aed3935", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:23.220Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e5c75c73-4260-4eb1-8fb7-2a786aed3935"} 2024-05-16T03:13:28.259Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2e9df526-cdfa-44db-8887-3ea3eb083351", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:28.349Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2e9df526-cdfa-44db-8887-3ea3eb083351"} 2024-05-16T03:13:33.405Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f67266f4-28b5-43f7-9867-8274e63240f3", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:33.484Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f67266f4-28b5-43f7-9867-8274e63240f3"} 2024-05-16T03:13:38.525Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "edda9854-3a88-4fb3-944c-8acb61de9f0b", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:38.620Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "edda9854-3a88-4fb3-944c-8acb61de9f0b"} 2024-05-16T03:13:38.707Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30a384a4-b571-47e7-9de2-e63b49f45aef", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:38.842Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30a384a4-b571-47e7-9de2-e63b49f45aef"} 2024-05-16T03:13:43.707Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53b63467-7c52-4df9-bab9-bcf16e2609cd", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:43.819Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "53b63467-7c52-4df9-bab9-bcf16e2609cd"} 2024-05-16T03:13:48.897Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "acdcd4b1-afb4-4c4f-acae-3fea2164a7b5", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:49.009Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "acdcd4b1-afb4-4c4f-acae-3fea2164a7b5"} 2024-05-16T03:13:54.094Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5a28aa62-ebd2-436d-8f84-64c52e8b797b", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:54.226Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5a28aa62-ebd2-436d-8f84-64c52e8b797b"} 2024-05-16T03:13:59.311Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d11cd3a0-380a-497f-a3f2-5626a5a7787d", "oldSize": 1, "newSize": 2} 2024-05-16T03:13:59.458Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d11cd3a0-380a-497f-a3f2-5626a5a7787d"} 2024-05-16T03:13:59.563Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30ee7e1e-b55c-4f25-921a-96592cddc73d", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:04.564Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "dc7ac34c-9a0a-41ec-a4f0-3a552a29d09c", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:10.827Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "544b4544-1a7a-4ff7-aefa-1149205be983", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:17.048Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "142a75ca-1af2-4dbb-a377-fb18b670893b", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:23.301Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "66aa0a13-1951-4f68-b1b7-4da35cb706ed", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:29.520Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5fcf3a7b-5fb0-4ad1-9bf5-47dc532af142", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:35.747Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1e03ada9-5d89-45d9-9534-db7b84111d42", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:41.975Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3db635e1-0d9d-4095-97bd-465d9c88e942", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:48.197Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "cac6500d-e24f-46e9-8a95-ac164973d7a5", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:49.597Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b04ac4f8-a91b-492a-8a21-7cda3ab0de95", "oldSize": 1, "newSize": 2} 2024-05-16T03:14:54.597Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "75c78bbe-8cf7-490c-bb46-dac27708cf68", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:00.847Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "60bba8ab-7868-4c29-8595-236a67fa1dee", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:07.381Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "19b48c18-8051-45c7-8d59-f927871369a9", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:13.583Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "68d72193-d58c-4dca-84ca-cd859b7c5691", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:19.802Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ff0d4503-f06f-4f2c-97e0-4bde52f663eb", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:26.511Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b1b0da28-81f8-4801-b47e-ae5929b386ee", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:32.903Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8da6d812-b670-435e-92b7-1538aea151ce", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:39.126Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "49d3234c-6ada-4c83-b81a-278d392a7528", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:45.368Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f633aa37-616d-4ff1-bfbc-38ba1fef9d47", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:51.598Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7c603c3c-352f-45ce-b8da-be98edbf8417", "oldSize": 1, "newSize": 2} 2024-05-16T03:15:57.836Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "aba62908-9a35-4c85-ad19-8f253a467596", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:04.048Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8b8d788c-229a-4f2d-84e2-f1938408ef25", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:05.403Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "499cd75d-d352-4fc1-bb21-b21f8346ccd2", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:06.846Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "499cd75d-d352-4fc1-bb21-b21f8346ccd2", "new version": "8.0.19-10"} 2024-05-16T03:16:07.154Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9c7fec04-ce87-4eb6-8846-1f5939aabb92", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:10.404Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5949ee55-2502-4d46-b48b-e681488557e9", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:16.698Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "67261520-cacb-423b-8749-75a4c2e14969", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:23.268Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6729c6ca-c5ca-46a1-953a-a153ad9dc647", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:29.585Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "190115f6-aa4d-4fb8-8aad-45b4b7e41bbe", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:35.875Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9f7718d2-e5ec-438e-b6eb-385979b986d5", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:42.151Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6b56c38b-745b-4d1b-b4f5-e8aa69cff604", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:48.425Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0d9e515a-4849-4ffb-a167-9c54cf32214b", "oldSize": 1, "newSize": 2} 2024-05-16T03:16:54.727Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "97e5d252-f8b7-4ae7-b0bd-e49ea85a7713", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:01.003Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "2213b1bd-c09b-4b97-a53c-e29144812752", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:07.393Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "916a8f69-2782-4913-9818-8cf26e126cd1", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:13.685Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "33deb80c-16ce-4dd1-9b8d-3401378218c0", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:19.999Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3132e00b-8665-4b0f-9a52-d65aa41ca9d7", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:26.571Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "84e927a5-4fea-4346-a91b-0463c1a3a5b2", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:32.171Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:33.376Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "name": "ensure-version/smart-update2-20475/smart-update", "schedule": "* * * * *"} 2024-05-16T03:17:33.447Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8696d653-5290-4e67-b7bc-4530d322927f", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:38.448Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e4fd272f-b105-4a34-a25c-10a8b2f5d1a9", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:44.733Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f78bf13d-f2ec-4e71-a2ef-46f312fa4f7c", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:50.994Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "592ecb51-8c65-4ad5-8ddb-83ef9782e7db", "oldSize": 1, "newSize": 2} 2024-05-16T03:17:57.255Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d8377afa-bb43-453a-b785-9be973e90905", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:03.505Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "23194a32-09d4-4db3-9411-57e7bd6d5842", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:09.916Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1433d24e-5b4b-4b0f-92db-0b950556a0a7", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:16.173Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ac5a0d11-72a8-431c-9e58-0954f8dff333", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:22.487Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b406eeb6-5c6c-433a-ae74-cd50d3b49068", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:28.779Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "57edcde0-0525-4cab-88e8-33af733046e4", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:33.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "endpoint": "http://version-service.smart-update2-20475.svc.cluster.local:11000"} 2024-05-16T03:18:33.000Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:33.034Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=38d3bbba-b391-4c8f-981e-ce3ba4e41ab0&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/latest?clusterWideEnabled=true&customResourceUid=38d3bbba-b391-4c8f-981e-ce3ba4e41ab0&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-05-16T03:18:35.066Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f27ebad0-c91d-4175-9e9d-fdc55ecb12b6", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:41.715Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "dcb1978a-d261-42df-b21f-d83f13356d93", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:48.007Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "335d4d8a-4b22-49c9-9bb8-73526c4209be", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:54.504Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "312aec24-6cd5-4586-98cb-2337e134267f", "oldSize": 1, "newSize": 2} 2024-05-16T03:18:57.971Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e47bffca-2cee-406f-9186-c53476af5ec8", "oldSize": 1, "newSize": 2} 2024-05-16T03:19:34.206Z INFO cluster is not found, deleting the job {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6447758d-3262-4f32-9f25-abd07cdddc14", "name": "ensure-version/smart-update2-20475/smart-update", "cluster": "smart-update", "namespace": "smart-update2-20475"} 2024-05-16T03:19:54.521Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "fea16bea-edcc-481f-8b38-ce5875cfb9dd", "oldSize": 1, "newSize": 2} 2024-05-16T03:19:54.667Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "fea16bea-edcc-481f-8b38-ce5875cfb9dd"} 2024-05-16T03:19:55.348Z INFO KubeAPIWarningLogger .metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: b5d46dd8-7e7c-45a3-96a3-685f5f897af2 2024-05-16T03:19:55.489Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "289fff7f-a14a-47bb-8232-d26fd4366652", "oldSize": 1, "newSize": 2} 2024-05-16T03:19:55.685Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "289fff7f-a14a-47bb-8232-d26fd4366652"} 2024-05-16T03:19:59.210Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "aeed629e-d623-49ec-9733-cc4dd6763b06", "oldSize": 1, "newSize": 2} 2024-05-16T03:19:59.335Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "aeed629e-d623-49ec-9733-cc4dd6763b06"} 2024-05-16T03:20:04.381Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "65ba1ef4-ce41-4960-98e1-c299df221dc9", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:04.467Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "65ba1ef4-ce41-4960-98e1-c299df221dc9"} 2024-05-16T03:20:09.508Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "19879541-9be2-4d6a-a727-c6552976854c", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:09.591Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "19879541-9be2-4d6a-a727-c6552976854c"} 2024-05-16T03:20:14.632Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f7e15f73-48a0-40cd-b0b0-7c7869eb20d9", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:14.747Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f7e15f73-48a0-40cd-b0b0-7c7869eb20d9"} 2024-05-16T03:20:19.783Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9ff897cb-0142-4020-89d7-4c20a36cb8e6", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:20.395Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9ff897cb-0142-4020-89d7-4c20a36cb8e6"} 2024-05-16T03:20:25.478Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e19b2e0a-ba90-4f6e-94b5-ab98f65720a2", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:25.558Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e19b2e0a-ba90-4f6e-94b5-ab98f65720a2"} 2024-05-16T03:20:30.591Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "36a6c6df-4248-4409-b195-d45a4973c7c7", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:30.707Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "36a6c6df-4248-4409-b195-d45a4973c7c7"} 2024-05-16T03:20:35.746Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "319b2c75-3ca8-411e-b11b-730cff6b267d", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:35.897Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "319b2c75-3ca8-411e-b11b-730cff6b267d"} 2024-05-16T03:20:40.938Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d95e0721-6421-4c09-8dad-68f79d41bcd5", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:41.025Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d95e0721-6421-4c09-8dad-68f79d41bcd5"} 2024-05-16T03:20:46.061Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3641337e-46c6-490e-8b45-ec7dca103728", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:46.152Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3641337e-46c6-490e-8b45-ec7dca103728"} 2024-05-16T03:20:51.233Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1a3a8bb8-8a31-4a7f-b516-501e1243c2f2", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:51.310Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1a3a8bb8-8a31-4a7f-b516-501e1243c2f2"} 2024-05-16T03:20:56.350Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0df43478-89ca-4aa6-a0dc-f5c42910e954", "oldSize": 1, "newSize": 2} 2024-05-16T03:20:56.440Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0df43478-89ca-4aa6-a0dc-f5c42910e954"} 2024-05-16T03:21:01.474Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8337d925-7ed4-4074-b327-cf99fc522d3b", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:01.553Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8337d925-7ed4-4074-b327-cf99fc522d3b"} 2024-05-16T03:21:06.586Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7b077633-8031-4198-9f06-1f4ac4383fee", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:06.700Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7b077633-8031-4198-9f06-1f4ac4383fee"} 2024-05-16T03:21:11.750Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6b5e5c06-ecc3-4722-a6be-106458b8c02a", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:11.880Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6b5e5c06-ecc3-4722-a6be-106458b8c02a"} 2024-05-16T03:21:11.979Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f720dd20-f64d-4e88-b84b-5ee64904e4a8", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:12.156Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f720dd20-f64d-4e88-b84b-5ee64904e4a8"} 2024-05-16T03:21:16.980Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30996308-a952-4d93-ac1f-d42fde90fb57", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:17.204Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "30996308-a952-4d93-ac1f-d42fde90fb57"} 2024-05-16T03:21:22.317Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a85f72b4-a5fe-4b4a-aff1-aa94c6e79761", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:22.444Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "a85f72b4-a5fe-4b4a-aff1-aa94c6e79761"} 2024-05-16T03:21:27.529Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8b77a727-79c6-40e8-a6e0-46a0f2d8132a", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:27.737Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8b77a727-79c6-40e8-a6e0-46a0f2d8132a"} 2024-05-16T03:21:32.922Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d9993895-a7d6-4ef0-8dd4-74c5a2a764ca", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:33.075Z INFO Waiting for HAProxy to be ready before smart update {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d9993895-a7d6-4ef0-8dd4-74c5a2a764ca"} 2024-05-16T03:21:33.182Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b6b2c838-b945-4940-8730-485f203073c3", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:38.183Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "87ab93c3-c7c3-4873-8311-8366bc3a7870", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:44.461Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "175906bd-87ed-453f-88de-6d874169d0be", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:50.696Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5d72b7f5-9522-4856-a8c6-4258903b962d", "oldSize": 1, "newSize": 2} 2024-05-16T03:21:56.913Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e6fc15d2-74f1-48e2-88f4-f0091e5c393c", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:03.168Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e615c236-8c89-443b-8a4d-ecc5fe218e46", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:09.433Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "25ac202e-a3b2-48d0-a2da-9cc20cc92652", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:15.635Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "1680f6da-aaf0-4ef3-a3ff-66461fae8dc3", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:22.049Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d5a814e6-d574-4f6c-b78b-530d685cce99", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:23.310Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "bedda93a-9560-4489-9a26-5d8f648534eb", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:28.310Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7662916b-450d-4e56-9c32-110296584c6d", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:34.539Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "183f82ac-f5ca-490a-934a-704c7cf5c5b5", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:40.790Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "958b344a-c2f9-4034-84f5-99056e4e6729", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:47.024Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9c12e5c3-a191-4a8d-a774-d31c05eb5f40", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:53.341Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "12a1feca-adda-4d41-9e0c-01c1d49ce7b5", "oldSize": 1, "newSize": 2} 2024-05-16T03:22:59.580Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "6f93b1be-b9f5-4296-bd9c-926796a3a4cb", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:05.853Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "9e72a802-238b-4c51-b2cf-7777d6e297af", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:12.229Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ee54253a-a76b-4061-8f59-fa838b6da36f", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:18.491Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b16ad823-ea52-4e83-bda8-d13c29643b5f", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:24.764Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7ff45c8d-a83d-4630-a5f6-002975150ea7", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:31.000Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f96a5cdf-d7d0-4208-ba26-3e7ce77c6e4d", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:37.221Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "21d0e050-7567-46d1-a54a-b8cb1beadb3b", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:38.589Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7dd19c23-94a4-4726-b589-5439ccabf4dc", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:39.835Z INFO update PXC version (fetched from db) {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "7dd19c23-94a4-4726-b589-5439ccabf4dc", "new version": "8.0.19-10"} 2024-05-16T03:23:39.936Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "67b4d11e-b1c6-4070-be91-288ee15f018f", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:43.590Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b2ae80f3-1443-40ef-a24d-b5dc1bfe9c9b", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:49.843Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "0d43ae29-0404-4dc7-9e7c-9b34a33826ba", "oldSize": 1, "newSize": 2} 2024-05-16T03:23:56.660Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "4912ffbd-b8f2-4985-8121-bd4a807a990f", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:02.951Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e23959ff-331c-4544-9bd8-8b56b4c3242a", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:09.244Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ddadceb5-eb24-4706-a17e-e51ef2fe111c", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:15.506Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8672f575-2deb-468f-b121-5b90421f388a", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:22.055Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d66bd15d-0446-46c2-9f1b-f0b348af33bf", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:28.361Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ca84bc86-f3af-4eb5-b1d2-329648ef4cbe", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:34.670Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e647ab83-fe42-496e-a574-f49d0bcc255c", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:41.318Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f5be17e9-4e11-411b-8d92-48a5238a2de1", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:47.603Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c1d2b036-a1e4-4181-be36-c6db5a1f43a3", "oldSize": 1, "newSize": 2} 2024-05-16T03:24:53.936Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "e3a176ba-e895-4049-b708-82e6f10254ab", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:00.309Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "ff5826a2-0366-4418-9c31-a61e327dda4e", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:05.444Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:06.968Z INFO add new job {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "name": "ensure-version/smart-update2-20475/smart-update", "schedule": "* * * * *"} 2024-05-16T03:25:07.080Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c029617b-876a-4ac5-bc6c-0c76a3485678", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:12.081Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "bbd9cd05-e788-497a-9afe-2d676f482835", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:18.354Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d504f2c1-4340-46c0-b794-ff93745c59ea", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:24.682Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "baf109be-dc45-425f-b61c-d77399cbbd23", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:31.004Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "23ead357-079f-4b7b-b8f0-e15b8f90f2f3", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:37.305Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "c1a225b8-a189-41ac-9dc9-8d7c18c34392", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:43.586Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "38a0ffe8-6841-4ed7-848d-abd3bb69c0a2", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:50.032Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "f1cced55-3a05-4a1f-8247-0b88b78e611c", "oldSize": 1, "newSize": 2} 2024-05-16T03:25:56.633Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "d1212435-3120-4286-9e46-ca601042c942", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:02.917Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "762da093-69a3-4f2d-b0fc-fa1b852f1670", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:06.000Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "endpoint": "http://version-service.smart-update2-20475.svc.cluster.local:11000"} 2024-05-16T03:26:06.000Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:06.041Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=b5d46dd8-7e7c-45a3-96a3-685f5f897af2&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=b5d46dd8-7e7c-45a3-96a3-685f5f897af2&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} 2024-05-16T03:26:09.183Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b0926e32-f8cf-4cca-8099-a79116b3f5db", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:15.473Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "5a2c6c7a-1aab-425b-b80c-dce3af8ee0f9", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:21.926Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "b92fccc2-f619-4472-b46f-6a88d19e51bc", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:28.204Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "3accad77-ce92-4b54-8eba-796528ca1105", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:34.492Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "cea6b180-f061-4f44-80a0-b51ad55968f4", "oldSize": 1, "newSize": 2} 2024-05-16T03:26:37.719Z INFO Setting safe defaults, updating HAProxy size {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "8e857521-aa41-46f0-bd1f-31acde5ce307", "oldSize": 1, "newSize": 2} 2024-05-16T03:27:06.019Z DEBUG Use version service endpoint {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "endpoint": "http://version-service.smart-update2-20475.svc.cluster.local:11000"} 2024-05-16T03:27:06.060Z ERROR failed to ensure version {"controller": "pxc-controller", "namespace": "smart-update2-20475", "name": "smart-update", "reconcileID": "735d6d86-a804-4bef-aa96-4c919c47cb70", "error": "failed to get new versions: failed to check version: Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=b5d46dd8-7e7c-45a3-96a3-685f5f897af2&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host", "errorVerbose": "Get \"http://version-service:11000/versions/v1/pxc-operator/9.9.9/percona%2Fpercona-xtradb-cluster:8.0.19-10.1?clusterWideEnabled=true&customResourceUid=b5d46dd8-7e7c-45a3-96a3-685f5f897af2&databaseVersion=8.0.19-10&kubeVersion=v1.26.15-gke.1300000&platform=kubernetes\": dial tcp: lookup version-service on 10.154.0.10:53: no such host\nfailed to check version\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).getNewVersions\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:266\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:283\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695\nfailed to get new versions\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).ensurePXCVersion\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:285\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:212\ngithub.com/robfig/cron/v3.FuncJob.Run\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136\ngithub.com/robfig/cron/v3.(*Cron).startJob.func1\n\t/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"} github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).scheduleEnsurePXCVersion.func1 github.com/robfig/cron/v3.(*Cron).startJob.func1 github.com/robfig/cron/v3.FuncJob.Run /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:136 /go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:222 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:261 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.0/pkg/internal/controller/controller.go:324 /go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/version.go:214 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 + kubectl get pxc --all-namespaces -o wide + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name was specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.3jtrj4r6wk ++ mktemp + local LAST_ERR=/tmp/tmp.alNp822QXE + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.3jtrj4r6wk No resources found + cat /tmp/tmp.alNp822QXE + rm /tmp/tmp.3jtrj4r6wk /tmp/tmp.alNp822QXE + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.owBW1L6zs7 ++ mktemp + local LAST_ERR=/tmp/tmp.yXjyJcyiyV + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.owBW1L6zs7 No resources found + cat /tmp/tmp.yXjyJcyiyV + rm /tmp/tmp.owBW1L6zs7 /tmp/tmp.yXjyJcyiyV + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.lOWcDn7XVw ++ mktemp + local LAST_ERR=/tmp/tmp.sfPhuhtscv + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.lOWcDn7XVw No resources found + cat /tmp/tmp.sfPhuhtscv + rm /tmp/tmp.lOWcDn7XVw /tmp/tmp.sfPhuhtscv + return 0 + kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook ++ mktemp + local LAST_OUT=/tmp/tmp.SY0gnXsw2h ++ mktemp + local LAST_ERR=/tmp/tmp.Vi2eZO9KaA + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook + exit_status=0 + set -e + '[' 0 '!=' 0 ']' + break + cat /tmp/tmp.SY0gnXsw2h validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted + cat /tmp/tmp.Vi2eZO9KaA + rm /tmp/tmp.SY0gnXsw2h /tmp/tmp.Vi2eZO9KaA + return 0 + kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml + : + '[' '!' -z '' ']' + '[' -n pxc-operator ']' + kubectl_bin delete --grace-period=0 --force=true namespace smart-update2-20475 + rm -rf /tmp/tmp.qA2QIzYgTk ++ mktemp + kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.2gCaDFbF8I + desc 'test passed' + set +o xtrace ----------------------------------------------------------------------------------- test passed ----------------------------------------------------------------------------------- + local LAST_OUT=/tmp/tmp.78Z17ij1nG ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.qhUCrIX41C + local exit_status=0 + local LAST_ERR=/tmp/tmp.vw1wvu83tY + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace smart-update2-20475 + for i in '$(seq 0 2)' + set +e + kubectl delete --grace-period=0 --force=true namespace pxc-operator