Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/logs/upgrade-consistency-8-0.log
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ main
+ create_infra upgrade-consistency-9051
+ local ns=upgrade-consistency-9051
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n upgrade-consistency-22838 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.MLTCQwUtBL
++ mktemp
+ local LAST_ERR=/tmp/tmp.BNSmdmXTcX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MLTCQwUtBL
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.BNSmdmXTcX
+ rm /tmp/tmp.MLTCQwUtBL /tmp/tmp.BNSmdmXTcX
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.Fq6sROsPep
++ mktemp
+ local LAST_ERR=/tmp/tmp.Iz0HwjuG0b
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Fq6sROsPep
No resources found
+ cat /tmp/tmp.Iz0HwjuG0b
+ rm /tmp/tmp.Fq6sROsPep /tmp/tmp.Iz0HwjuG0b
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.rE2Lx5E0Ph
++ mktemp
+ local LAST_ERR=/tmp/tmp.dtt7IFRHvU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.rE2Lx5E0Ph
No resources found
+ cat /tmp/tmp.dtt7IFRHvU
+ rm /tmp/tmp.rE2Lx5E0Ph /tmp/tmp.dtt7IFRHvU
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.Wy9YK8MtUZ
++ mktemp
+ local LAST_OUT=/tmp/tmp.ujwNuUJgbI
++ mktemp
+ awk '{print$1}'
+ local LAST_ERR=/tmp/tmp.SNHoPzloVV
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.8NFAJnzTf5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ujwNuUJgbI
+ cat /tmp/tmp.8NFAJnzTf5
+ rm /tmp/tmp.ujwNuUJgbI /tmp/tmp.8NFAJnzTf5
+ return 0
namespace "upgrade-consistency-22838" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Wy9YK8MtUZ
namespace "pxc-operator" deleted
+ cat /tmp/tmp.SNHoPzloVV
+ rm /tmp/tmp.Wy9YK8MtUZ /tmp/tmp.SNHoPzloVV
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'namespace/pxc-operator - '
namespace/pxc-operator - + set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.5Lf6BQftTe
++ mktemp
+ local LAST_ERR=/tmp/tmp.PPVpjqH7yK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5Lf6BQftTe
namespace/pxc-operator created
+ cat /tmp/tmp.PPVpjqH7yK
+ rm /tmp/tmp.5Lf6BQftTe /tmp/tmp.PPVpjqH7yK
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PQTDhViHnH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MA0ieBdtgZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PQTDhViHnH
++ cat /tmp/tmp.MA0ieBdtgZ
++ rm /tmp/tmp.PQTDhViHnH /tmp/tmp.MA0ieBdtgZ
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.ms9WafshBh
++ mktemp
+ local LAST_ERR=/tmp/tmp.jrMOIkyTvA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ms9WafshBh
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8" modified.
+ cat /tmp/tmp.jrMOIkyTvA
+ rm /tmp/tmp.ms9WafshBh /tmp/tmp.jrMOIkyTvA
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1Er7IOUPsb
++ mktemp
+ local LAST_ERR=/tmp/tmp.e6MOtKyU88
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1Er7IOUPsb
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.e6MOtKyU88
+ rm /tmp/tmp.1Er7IOUPsb /tmp/tmp.e6MOtKyU88
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/deploy/cw-rbac.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gj9dVYwx6r
++ mktemp
+ local LAST_ERR=/tmp/tmp.MSPzwW5gil
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Gj9dVYwx6r
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.MSPzwW5gil
+ rm /tmp/tmp.Gj9dVYwx6r /tmp/tmp.MSPzwW5gil
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1746-ef1f34c2^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/deploy/cw-operator.yaml
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.ibJXIbj08j
++ mktemp
+ local LAST_ERR=/tmp/tmp.1lUnT0Z73S
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ibJXIbj08j
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.1lUnT0Z73S
+ rm /tmp/tmp.ibJXIbj08j /tmp/tmp.1lUnT0Z73S
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.G169jvBTO0
++ mktemp
+ local LAST_ERR=/tmp/tmp.eJfupNqd5l
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.G169jvBTO0
pod/percona-xtradb-cluster-operator-5bfd85498-fjkc8 condition met
+ cat /tmp/tmp.eJfupNqd5l
+ rm /tmp/tmp.G169jvBTO0 /tmp/tmp.eJfupNqd5l
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.VsLpUSMvRa
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LVVEyr9Swm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.VsLpUSMvRa
++ cat /tmp/tmp.LVVEyr9Swm
++ rm /tmp/tmp.VsLpUSMvRa /tmp/tmp.LVVEyr9Swm
++ return 0
+ wait_pod percona-xtradb-cluster-operator-5bfd85498-fjkc8 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-5bfd85498-fjkc8
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-5bfd85498-fjkc8
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-5bfd85498-fjkc8 condition met
percona-xtradb-cluster-operator-5bfd85498-fjkc8.Ok
+ sleep 3
+ create_namespace upgrade-consistency-9051
+ local namespace=upgrade-consistency-9051
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-9051'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-9051
-----------------------------------------------------------------------------------

+ xargs kubectl delete ns
+ kubectl_bin delete namespace upgrade-consistency-9051
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.TfUA3OFT8i
+ local LAST_OUT=/tmp/tmp.Ws4jFAjipf
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.tyzF0F9IEp
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.VCdrLUO2Fg
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-9051
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TfUA3OFT8i
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-9051
+ cat /tmp/tmp.tyzF0F9IEp
+ rm /tmp/tmp.TfUA3OFT8i /tmp/tmp.tyzF0F9IEp
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-9051
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.Ws4jFAjipf
+ cat /tmp/tmp.VCdrLUO2Fg
Error from server (NotFound): namespaces "upgrade-consistency-9051" not found
+ rm /tmp/tmp.Ws4jFAjipf /tmp/tmp.VCdrLUO2Fg
+ return 1
+ :
+ wait_for_delete namespace/upgrade-consistency-9051
+ local res=namespace/upgrade-consistency-9051
+ echo -n 'namespace/upgrade-consistency-9051 - '
namespace/upgrade-consistency-9051 - + set +o xtrace
Error from server (NotFound): namespaces "upgrade-consistency-9051" not found
+ desc 'create namespace upgrade-consistency-9051'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-9051
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-9051
++ mktemp
+ local LAST_OUT=/tmp/tmp.HjugI5dX9O
++ mktemp
+ local LAST_ERR=/tmp/tmp.t4tt9pKHR5
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-consistency-9051
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.HjugI5dX9O
namespace/upgrade-consistency-9051 created
+ cat /tmp/tmp.t4tt9pKHR5
+ rm /tmp/tmp.HjugI5dX9O /tmp/tmp.t4tt9pKHR5
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GTBQyNXSRk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dvxJScobCR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GTBQyNXSRk
++ cat /tmp/tmp.dvxJScobCR
++ rm /tmp/tmp.GTBQyNXSRk /tmp/tmp.dvxJScobCR
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8 --namespace=upgrade-consistency-9051
++ mktemp
+ local LAST_OUT=/tmp/tmp.DpmLQXATEv
++ mktemp
+ local LAST_ERR=/tmp/tmp.yuuVRQtOii
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8 --namespace=upgrade-consistency-9051
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DpmLQXATEv
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1746-ef1f34c2-1-cluster8" modified.
+ cat /tmp/tmp.yuuVRQtOii
+ rm /tmp/tmp.DpmLQXATEv /tmp/tmp.yuuVRQtOii
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.MPNMmVNkMk
++ mktemp
+ local LAST_ERR=/tmp/tmp.UZQYLAgweg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MPNMmVNkMk
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.UZQYLAgweg
+ rm /tmp/tmp.MPNMmVNkMk /tmp/tmp.UZQYLAgweg
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1746-ef1f34c2#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-consistency-9051~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_OUT=/tmp/tmp.YYw8CVWOck
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.cfjdFxjJmb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YYw8CVWOck
deployment.apps/pxc-client created
+ cat /tmp/tmp.cfjdFxjJmb
+ rm /tmp/tmp.YYw8CVWOck /tmp/tmp.cfjdFxjJmb
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.1AIQmRtWN4
++ mktemp
+ local LAST_ERR=/tmp/tmp.n5pKNcruyN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1AIQmRtWN4
secret/my-cluster-secrets created
+ cat /tmp/tmp.n5pKNcruyN
+ rm /tmp/tmp.1AIQmRtWN4 /tmp/tmp.n5pKNcruyN
+ return 0
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/some-name.yml
+ yq eval '.spec.crVersion = "1.12.0"' -
+ kubectl_bin apply -f -
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.ARYGGkKMS0
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-consistency-9051~
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1746-ef1f34c2#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/conf/some-name.yml
++ mktemp
+ local LAST_ERR=/tmp/tmp.MzUK31HQOO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ARYGGkKMS0
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.MzUK31HQOO
+ rm /tmp/tmp.ARYGGkKMS0 /tmp/tmp.MzUK31HQOO
+ return 0
+ desc 'test 1.12.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.12.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.12.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.C33W10RmuO
++ mktemp
+ local LAST_ERR=/tmp/tmp.dWun8swD5i
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.12.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.C33W10RmuO
perconaxtradbcluster.pxc.percona.com/some-name patched (no change)
+ cat /tmp/tmp.dWun8swD5i
+ rm /tmp/tmp.C33W10RmuO /tmp/tmp.dWun8swD5i
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LQs6TJUha4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9gDg0EkgV2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LQs6TJUha4
++ cat /tmp/tmp.9gDg0EkgV2
++ rm /tmp/tmp.LQs6TJUha4 /tmp/tmp.9gDg0EkgV2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SOdfxAPgZ4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bJtYcYVBso
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SOdfxAPgZ4
++ cat /tmp/tmp.bJtYcYVBso
++ rm /tmp/tmp.SOdfxAPgZ4 /tmp/tmp.bJtYcYVBso
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jy2tUxAsNt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SQcpfxdjWU
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jy2tUxAsNt
++ cat /tmp/tmp.SQcpfxdjWU
++ rm /tmp/tmp.jy2tUxAsNt /tmp/tmp.SQcpfxdjWU
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.S7r0Tk18Es
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RVP6blAdIs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.S7r0Tk18Es
++ cat /tmp/tmp.RVP6blAdIs
++ rm /tmp/tmp.S7r0Tk18Es /tmp/tmp.RVP6blAdIs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HwuD7CYwmA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SCc2UFVh9L
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HwuD7CYwmA
++ cat /tmp/tmp.SCc2UFVh9L
++ rm /tmp/tmp.HwuD7CYwmA /tmp/tmp.SCc2UFVh9L
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.go4luxwHAp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mxl5ILvCeW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.go4luxwHAp
++ cat /tmp/tmp.mxl5ILvCeW
++ rm /tmp/tmp.go4luxwHAp /tmp/tmp.mxl5ILvCeW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vUtnfie9JF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KH6Ojd4OvZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vUtnfie9JF
++ cat /tmp/tmp.KH6Ojd4OvZ
++ rm /tmp/tmp.vUtnfie9JF /tmp/tmp.KH6Ojd4OvZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uYtNsoBHUA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O7d7iNzDDh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uYtNsoBHUA
++ cat /tmp/tmp.O7d7iNzDDh
++ rm /tmp/tmp.uYtNsoBHUA /tmp/tmp.O7d7iNzDDh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rrbczJdlAL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.wME2FXuhik
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rrbczJdlAL
++ cat /tmp/tmp.wME2FXuhik
++ rm /tmp/tmp.rrbczJdlAL /tmp/tmp.wME2FXuhik
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 8 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oQOvQwwkNJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.SsTDmwKext
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oQOvQwwkNJ
++ cat /tmp/tmp.SsTDmwKext
++ rm /tmp/tmp.oQOvQwwkNJ /tmp/tmp.SsTDmwKext
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 9 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.svbW88tJwV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rOiueCthz9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.svbW88tJwV
++ cat /tmp/tmp.rOiueCthz9
++ rm /tmp/tmp.svbW88tJwV /tmp/tmp.rOiueCthz9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 10 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Dw4jFz9l4q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VpDx1OeFTG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Dw4jFz9l4q
++ cat /tmp/tmp.VpDx1OeFTG
++ rm /tmp/tmp.Dw4jFz9l4q /tmp/tmp.VpDx1OeFTG
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FcFJRnovQ6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LwAkQppkxr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FcFJRnovQ6
++ cat /tmp/tmp.LwAkQppkxr
++ rm /tmp/tmp.FcFJRnovQ6 /tmp/tmp.LwAkQppkxr
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.wl1pmTmpqV
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.BG49sOQgK3
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.wl1pmTmpqV
+++++ cat /tmp/tmp.BG49sOQgK3
+++++ rm /tmp/tmp.wl1pmTmpqV /tmp/tmp.BG49sOQgK3
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.FLlBFiOVrW
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.q2vEbHcJxn
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.FLlBFiOVrW
+++++ cat /tmp/tmp.q2vEbHcJxn
+++++ rm /tmp/tmp.FLlBFiOVrW /tmp/tmp.q2vEbHcJxn
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.He1ZQcygZj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oE4T3ShoK6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.He1ZQcygZj
++ cat /tmp/tmp.oE4T3ShoK6
++ rm /tmp/tmp.He1ZQcygZj /tmp/tmp.oE4T3ShoK6
++ return 0
+ [[ 2 == \2 ]]
+ wait_for_sts_generation some-name-pxc 1 1
+ local name=some-name-pxc
+ local desired_generation=1
+ local size=1
+ set +o xtrace
Waiting for statefulset readyness
+ desc 'compare k8s objects'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare k8s objects
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-pxc -1120
+ local resource=service/some-name-pxc
+ local postfix=-1120
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ desc 'compare service/some-name-pxc--1120'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-pxc--1120
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120-eks.yml ']'
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ kubectl_bin get -o yaml service/some-name-pxc
++ mktemp
+ local LAST_OUT=/tmp/tmp.nh2MYfmBbI
++ mktemp
+ local LAST_ERR=/tmp/tmp.Ys6luUYRjf
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nh2MYfmBbI
+ cat /tmp/tmp.Ys6luUYRjf
+ rm /tmp/tmp.nh2MYfmBbI /tmp/tmp.Ys6luUYRjf
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1120.yml /tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ compare_kubectl service/some-name-proxysql -1120
+ local resource=service/some-name-proxysql
+ local postfix=-1120
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ desc 'compare service/some-name-proxysql--1120'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-proxysql--1120
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.29'
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.24'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.21'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-proxysql
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.SmaKOqF0Za
++ mktemp
+ local LAST_ERR=/tmp/tmp.lHYsTI4Xot
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SmaKOqF0Za
+ cat /tmp/tmp.lHYsTI4Xot
+ rm /tmp/tmp.SmaKOqF0Za /tmp/tmp.lHYsTI4Xot
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1120.yml /tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ compare_kubectl statefulset/some-name-pxc -1120
+ local resource=statefulset/some-name-pxc
+ local postfix=-1120
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ desc 'compare statefulset/some-name-pxc--1120'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-pxc--1120
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.185a9u8Ygr
++ mktemp
+ local LAST_ERR=/tmp/tmp.csgjslmGIv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.185a9u8Ygr
+ cat /tmp/tmp.csgjslmGIv
+ rm /tmp/tmp.185a9u8Ygr /tmp/tmp.csgjslmGIv
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1120.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ compare_kubectl statefulset/some-name-proxysql -1120
+ local resource=statefulset/some-name-proxysql
+ local postfix=-1120
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'compare statefulset/some-name-proxysql--1120'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-proxysql--1120
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.24'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.21'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120-eks.yml ']'
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ kubectl_bin get -o yaml statefulset/some-name-proxysql
++ mktemp
+ local LAST_OUT=/tmp/tmp.B8uoUeyllg
++ mktemp
+ local LAST_ERR=/tmp/tmp.KdUIeJpJVE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.B8uoUeyllg
+ cat /tmp/tmp.KdUIeJpJVE
+ rm /tmp/tmp.B8uoUeyllg /tmp/tmp.KdUIeJpJVE
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1120.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'test 1.13.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.13.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.13.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.YyWRwoVQvA
++ mktemp
+ local LAST_ERR=/tmp/tmp.7dfgg5iBn6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.13.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YyWRwoVQvA
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.7dfgg5iBn6
+ rm /tmp/tmp.YyWRwoVQvA /tmp/tmp.7dfgg5iBn6
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zYItLArOaA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.C7XPVzP5M9
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zYItLArOaA
++ cat /tmp/tmp.C7XPVzP5M9
++ rm /tmp/tmp.zYItLArOaA /tmp/tmp.C7XPVzP5M9
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WWTybv0Kuc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.npBrv0YIt0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WWTybv0Kuc
++ cat /tmp/tmp.npBrv0YIt0
++ rm /tmp/tmp.WWTybv0Kuc /tmp/tmp.npBrv0YIt0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DnrN23PbOK
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sciZ48UAB4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DnrN23PbOK
++ cat /tmp/tmp.sciZ48UAB4
++ rm /tmp/tmp.DnrN23PbOK /tmp/tmp.sciZ48UAB4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yReL0gUIcR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cc2EtCgf3y
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yReL0gUIcR
++ cat /tmp/tmp.cc2EtCgf3y
++ rm /tmp/tmp.yReL0gUIcR /tmp/tmp.cc2EtCgf3y
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IV5X1zTVCN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6e1griZvCT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IV5X1zTVCN
++ cat /tmp/tmp.6e1griZvCT
++ rm /tmp/tmp.IV5X1zTVCN /tmp/tmp.6e1griZvCT
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LnQ5tEM7Gb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bSy84ZxQuW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LnQ5tEM7Gb
++ cat /tmp/tmp.bSy84ZxQuW
++ rm /tmp/tmp.LnQ5tEM7Gb /tmp/tmp.bSy84ZxQuW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gJUmeg8R7m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BbEFfkFje0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gJUmeg8R7m
++ cat /tmp/tmp.BbEFfkFje0
++ rm /tmp/tmp.gJUmeg8R7m /tmp/tmp.BbEFfkFje0
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.K9tr2xPsi3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3RdRnEWfOd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.K9tr2xPsi3
++ cat /tmp/tmp.3RdRnEWfOd
++ rm /tmp/tmp.K9tr2xPsi3 /tmp/tmp.3RdRnEWfOd
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ubDxBUFJcq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.X4CwZmSg2O
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ubDxBUFJcq
++ cat /tmp/tmp.X4CwZmSg2O
++ rm /tmp/tmp.ubDxBUFJcq /tmp/tmp.X4CwZmSg2O
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.yHxdIBoxi6
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.92CUgkyk7I
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.yHxdIBoxi6
+++++ cat /tmp/tmp.92CUgkyk7I
+++++ rm /tmp/tmp.yHxdIBoxi6 /tmp/tmp.92CUgkyk7I
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.2rN0M6p17Y
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.p2KXtQnhRh
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.2rN0M6p17Y
+++++ cat /tmp/tmp.p2KXtQnhRh
+++++ rm /tmp/tmp.2rN0M6p17Y /tmp/tmp.p2KXtQnhRh
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ACNHrzGukT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oPUliewlcD
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ACNHrzGukT
++ cat /tmp/tmp.oPUliewlcD
++ rm /tmp/tmp.ACNHrzGukT /tmp/tmp.oPUliewlcD
++ return 0
+ [[ 2 == \2 ]]
+ wait_for_sts_generation some-name-pxc 2 1
+ local name=some-name-pxc
+ local desired_generation=2
+ local size=1
+ set +o xtrace
Waiting for statefulset readyness
+ desc 'compare k8s objects'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare k8s objects
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-pxc -1130
+ local resource=service/some-name-pxc
+ local postfix=-1130
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ desc 'compare service/some-name-pxc--1130'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-pxc--1130
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-pxc
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.nsOqkaJnIJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.Zs3fSSWzUZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nsOqkaJnIJ
+ cat /tmp/tmp.Zs3fSSWzUZ
+ rm /tmp/tmp.nsOqkaJnIJ /tmp/tmp.Zs3fSSWzUZ
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1130.yml /tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ compare_kubectl service/some-name-proxysql -1130
+ local resource=service/some-name-proxysql
+ local postfix=-1130
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ desc 'compare service/some-name-proxysql--1130'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-proxysql--1130
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.24'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.lC285oPnNc
++ mktemp
+ local LAST_ERR=/tmp/tmp.eL3mSEdlZq
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lC285oPnNc
+ cat /tmp/tmp.eL3mSEdlZq
+ rm /tmp/tmp.lC285oPnNc /tmp/tmp.eL3mSEdlZq
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1130.yml /tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ compare_kubectl statefulset/some-name-pxc -1130
+ local resource=statefulset/some-name-pxc
+ local postfix=-1130
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ desc 'compare statefulset/some-name-pxc--1130'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-pxc--1130
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.59wcRp9KOi
++ mktemp
+ local LAST_ERR=/tmp/tmp.3ao7xzXt6t
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.59wcRp9KOi
+ cat /tmp/tmp.3ao7xzXt6t
+ rm /tmp/tmp.59wcRp9KOi /tmp/tmp.3ao7xzXt6t
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1130.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ compare_kubectl statefulset/some-name-proxysql -1130
+ local resource=statefulset/some-name-proxysql
+ local postfix=-1130
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'compare statefulset/some-name-proxysql--1130'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-proxysql--1130
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.vZn3B8KitD
++ mktemp
+ local LAST_ERR=/tmp/tmp.MgNqt07MNv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vZn3B8KitD
+ cat /tmp/tmp.MgNqt07MNv
+ rm /tmp/tmp.vZn3B8KitD /tmp/tmp.MgNqt07MNv
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1130.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'test 1.14.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.14.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.14.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.jbPyo1vmUv
++ mktemp
+ local LAST_ERR=/tmp/tmp.3DOQEQ8wj3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.14.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jbPyo1vmUv
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.3DOQEQ8wj3
+ rm /tmp/tmp.jbPyo1vmUv /tmp/tmp.3DOQEQ8wj3
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0F6Sh20Bb4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cCfFJGhcGn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0F6Sh20Bb4
++ cat /tmp/tmp.cCfFJGhcGn
++ rm /tmp/tmp.0F6Sh20Bb4 /tmp/tmp.cCfFJGhcGn
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3M7WveSZPs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.eBhHKJ0wjZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3M7WveSZPs
++ cat /tmp/tmp.eBhHKJ0wjZ
++ rm /tmp/tmp.3M7WveSZPs /tmp/tmp.eBhHKJ0wjZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6XA0FX5ITq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ri2bMN7G1H
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6XA0FX5ITq
++ cat /tmp/tmp.Ri2bMN7G1H
++ rm /tmp/tmp.6XA0FX5ITq /tmp/tmp.Ri2bMN7G1H
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.u4zupEBkqz
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sZrET2dtaM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.u4zupEBkqz
++ cat /tmp/tmp.sZrET2dtaM
++ rm /tmp/tmp.u4zupEBkqz /tmp/tmp.sZrET2dtaM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FHWjk5Lwfe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.WuLxIq7Egl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.FHWjk5Lwfe
++ cat /tmp/tmp.WuLxIq7Egl
++ rm /tmp/tmp.FHWjk5Lwfe /tmp/tmp.WuLxIq7Egl
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fWozOWvvdM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VRbBmf3ln6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.fWozOWvvdM
++ cat /tmp/tmp.VRbBmf3ln6
++ rm /tmp/tmp.fWozOWvvdM /tmp/tmp.VRbBmf3ln6
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DHJU24FPJ2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nAN6nCZRDo
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.DHJU24FPJ2
++ cat /tmp/tmp.nAN6nCZRDo
++ rm /tmp/tmp.DHJU24FPJ2 /tmp/tmp.nAN6nCZRDo
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xK2uIzBHfm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OUkhYGMBK0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xK2uIzBHfm
++ cat /tmp/tmp.OUkhYGMBK0
++ rm /tmp/tmp.xK2uIzBHfm /tmp/tmp.OUkhYGMBK0
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.G4yPrwmgpE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ve1CqhunqA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.G4yPrwmgpE
++ cat /tmp/tmp.Ve1CqhunqA
++ rm /tmp/tmp.G4yPrwmgpE /tmp/tmp.Ve1CqhunqA
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.FbxQXXJ6Xs
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.oDh20P49mP
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.FbxQXXJ6Xs
+++++ cat /tmp/tmp.oDh20P49mP
+++++ rm /tmp/tmp.FbxQXXJ6Xs /tmp/tmp.oDh20P49mP
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.H1jHg8i648
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.TUFyxHqA9Z
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.H1jHg8i648
+++++ cat /tmp/tmp.TUFyxHqA9Z
+++++ rm /tmp/tmp.H1jHg8i648 /tmp/tmp.TUFyxHqA9Z
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7s4BMhR8UH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OvK7k6fyxA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7s4BMhR8UH
++ cat /tmp/tmp.OvK7k6fyxA
++ rm /tmp/tmp.7s4BMhR8UH /tmp/tmp.OvK7k6fyxA
++ return 0
+ [[ 2 == \2 ]]
+ wait_for_sts_generation some-name-pxc 3 1
+ local name=some-name-pxc
+ local desired_generation=3
+ local size=1
+ set +o xtrace
Waiting for statefulset readyness
+ desc 'compare k8s objects'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare k8s objects
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-pxc -1140
+ local resource=service/some-name-pxc
+ local postfix=-1140
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ desc 'compare service/some-name-pxc--1140'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-pxc--1140
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-pxc
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.2s6UBspumD
++ mktemp
+ local LAST_ERR=/tmp/tmp.VbsRScl8rM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2s6UBspumD
+ cat /tmp/tmp.VbsRScl8rM
+ rm /tmp/tmp.2s6UBspumD /tmp/tmp.VbsRScl8rM
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1140.yml /tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ compare_kubectl service/some-name-proxysql -1140
+ local resource=service/some-name-proxysql
+ local postfix=-1140
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ desc 'compare service/some-name-proxysql--1140'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-proxysql--1140
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.29'
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.21'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140-eks.yml ']'
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ kubectl_bin get -o yaml service/some-name-proxysql
++ mktemp
+ local LAST_OUT=/tmp/tmp.xC1zWZjH1c
++ mktemp
+ local LAST_ERR=/tmp/tmp.niFK9rtOBb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xC1zWZjH1c
+ cat /tmp/tmp.niFK9rtOBb
+ rm /tmp/tmp.xC1zWZjH1c /tmp/tmp.niFK9rtOBb
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1140.yml /tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ compare_kubectl statefulset/some-name-pxc -1140
+ local resource=statefulset/some-name-pxc
+ local postfix=-1140
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ desc 'compare statefulset/some-name-pxc--1140'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-pxc--1140
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.dOzTCcx4tU
++ mktemp
+ local LAST_ERR=/tmp/tmp.yXyCuzbFhg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dOzTCcx4tU
+ cat /tmp/tmp.yXyCuzbFhg
+ rm /tmp/tmp.dOzTCcx4tU /tmp/tmp.yXyCuzbFhg
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1140.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ compare_kubectl statefulset/some-name-proxysql -1140
+ local resource=statefulset/some-name-proxysql
+ local postfix=-1140
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'compare statefulset/some-name-proxysql--1140'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-proxysql--1140
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-proxysql
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fshTvaSkgB
++ mktemp
+ local LAST_ERR=/tmp/tmp.t3W9y53hwt
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.fshTvaSkgB
+ cat /tmp/tmp.t3W9y53hwt
+ rm /tmp/tmp.fshTvaSkgB /tmp/tmp.t3W9y53hwt
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1140.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'test 1.15.0'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.15.0
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.15.0"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.H8RiQeMd3c
++ mktemp
+ local LAST_ERR=/tmp/tmp.AoJ47Iikgj
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.15.0"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.H8RiQeMd3c
perconaxtradbcluster.pxc.percona.com/some-name patched
+ cat /tmp/tmp.AoJ47Iikgj
+ rm /tmp/tmp.H8RiQeMd3c /tmp/tmp.AoJ47Iikgj
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.IEpxIkXN38
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xUcnNme1Zb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.IEpxIkXN38
++ cat /tmp/tmp.xUcnNme1Zb
++ rm /tmp/tmp.IEpxIkXN38 /tmp/tmp.xUcnNme1Zb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dsIE1MoYVh
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gvecDV0gGc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dsIE1MoYVh
++ cat /tmp/tmp.gvecDV0gGc
++ rm /tmp/tmp.dsIE1MoYVh /tmp/tmp.gvecDV0gGc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8bIPLDtRWS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Mmm5rmk6bO
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8bIPLDtRWS
++ cat /tmp/tmp.Mmm5rmk6bO
++ rm /tmp/tmp.8bIPLDtRWS /tmp/tmp.Mmm5rmk6bO
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Us1lc6N6bZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RalHcoNFcM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Us1lc6N6bZ
++ cat /tmp/tmp.RalHcoNFcM
++ rm /tmp/tmp.Us1lc6N6bZ /tmp/tmp.RalHcoNFcM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.getZQiDqmX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xOOuJCtDgM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.getZQiDqmX
++ cat /tmp/tmp.xOOuJCtDgM
++ rm /tmp/tmp.getZQiDqmX /tmp/tmp.xOOuJCtDgM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rHWKfnJqpN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zAkwHBa8go
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.rHWKfnJqpN
++ cat /tmp/tmp.zAkwHBa8go
++ rm /tmp/tmp.rHWKfnJqpN /tmp/tmp.zAkwHBa8go
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1MPnJCBeyI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ymS7a8PMOM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1MPnJCBeyI
++ cat /tmp/tmp.ymS7a8PMOM
++ rm /tmp/tmp.1MPnJCBeyI /tmp/tmp.ymS7a8PMOM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HONZA9zaBo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JrN8iRrTU6
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.HONZA9zaBo
++ cat /tmp/tmp.JrN8iRrTU6
++ rm /tmp/tmp.HONZA9zaBo /tmp/tmp.JrN8iRrTU6
++ return 0
+ [[ ready == \r\e\a\d\y ]]
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.pxc.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LAoGyjPYvH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RswdGqZ9Rv
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.pxc.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LAoGyjPYvH
++ cat /tmp/tmp.RswdGqZ9Rv
++ rm /tmp/tmp.LAoGyjPYvH /tmp/tmp.RswdGqZ9Rv
++ return 0
+ [[ 3 == \3 ]]
+++ get_proxy_engine some-name
+++ local cluster_name=some-name
++++ get_proxy some-name
++++ local target_cluster=some-name
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.dTvNdDguua
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.dDJp8Q6r7c
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.dTvNdDguua
+++++ cat /tmp/tmp.dDJp8Q6r7c
+++++ rm /tmp/tmp.dTvNdDguua /tmp/tmp.dDJp8Q6r7c
+++++ return 0
++++ [[ '' == \t\r\u\e ]]
+++++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++++ mktemp
+++++ local LAST_OUT=/tmp/tmp.wWMmU3roND
++++++ mktemp
+++++ local LAST_ERR=/tmp/tmp.2CMm8QLoSj
+++++ local exit_status=0
++++++ seq 0 2
+++++ for i in '$(seq 0 2)'
+++++ set +e
+++++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++++ exit_status=0
+++++ set -e
+++++ '[' 0 '!=' 0 ']'
+++++ break
+++++ cat /tmp/tmp.wWMmU3roND
+++++ cat /tmp/tmp.2CMm8QLoSj
+++++ rm /tmp/tmp.wWMmU3roND /tmp/tmp.2CMm8QLoSj
+++++ return 0
++++ [[ true == \t\r\u\e ]]
++++ echo some-name-proxysql
++++ return
+++ local cluster_proxy=some-name-proxysql
+++ echo proxysql
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zxCI3eb6HL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CYuLbkcRWQ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.proxysql.ready}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zxCI3eb6HL
++ cat /tmp/tmp.CYuLbkcRWQ
++ rm /tmp/tmp.zxCI3eb6HL /tmp/tmp.CYuLbkcRWQ
++ return 0
+ [[ 2 == \2 ]]
+ wait_for_sts_generation some-name-pxc 4 1
+ local name=some-name-pxc
+ local desired_generation=4
+ local size=1
+ set +o xtrace
Waiting for statefulset readyness
+ desc 'compare k8s objects'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare k8s objects
-----------------------------------------------------------------------------------

+ compare_kubectl service/some-name-pxc -1150
+ local resource=service/some-name-pxc
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ desc 'compare service/some-name-pxc--1150'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-pxc--1150
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.Pcboe0JKeu
++ mktemp
+ local LAST_ERR=/tmp/tmp.rj3TYnOhs7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Pcboe0JKeu
+ cat /tmp/tmp.rj3TYnOhs7
+ rm /tmp/tmp.Pcboe0JKeu /tmp/tmp.rj3TYnOhs7
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-pxc-1150.yml /tmp/tmp.NNcSZUPFDP/service_some-name-pxc.yml
+ compare_kubectl service/some-name-proxysql -1150
+ local resource=service/some-name-proxysql
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ desc 'compare service/some-name-proxysql--1150'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare service/some-name-proxysql--1150
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150-eks.yml ']'
+ kubectl_bin get -o yaml service/some-name-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.845x67gOwx
++ mktemp
+ local LAST_ERR=/tmp/tmp.teoWFkRBbY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml service/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.845x67gOwx
+ cat /tmp/tmp.teoWFkRBbY
+ rm /tmp/tmp.845x67gOwx /tmp/tmp.teoWFkRBbY
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/service_some-name-proxysql-1150.yml /tmp/tmp.NNcSZUPFDP/service_some-name-proxysql.yml
+ compare_kubectl statefulset/some-name-pxc -1150
+ local resource=statefulset/some-name-pxc
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ desc 'compare statefulset/some-name-pxc--1150'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-pxc--1150
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ bc -l
++ echo '1.26 >= 1.22'
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-pxc
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.3MtxER39Zj
++ mktemp
+ local LAST_ERR=/tmp/tmp.hYB5opDFf8
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-pxc
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3MtxER39Zj
+ cat /tmp/tmp.hYB5opDFf8
+ rm /tmp/tmp.3MtxER39Zj /tmp/tmp.hYB5opDFf8
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-pxc-1150.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-pxc.yml
+ compare_kubectl statefulset/some-name-proxysql -1150
+ local resource=statefulset/some-name-proxysql
+ local postfix=-1150
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150.yml
+ local new_result=/tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ desc 'compare statefulset/some-name-proxysql--1150'
+ set +o xtrace


-----------------------------------------------------------------------------------
compare statefulset/some-name-proxysql--1150
-----------------------------------------------------------------------------------

+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-eks.yml ']'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-80.yml ']'
+ version_gt 1.29
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.29'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.27
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.27'
++ bc -l
+ '[' 0 -eq 1 ']'
+ return 1
+ version_gt 1.24
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.24'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-k124.yml ']'
+ version_gt 1.22
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.22'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-k122.yml ']'
+ version_gt 1.21
+ desc 'return true if kubernetes version equal or greater than desired'
+ set +o xtrace


-----------------------------------------------------------------------------------
return true if kubernetes version equal or greater than desired
-----------------------------------------------------------------------------------

++ echo '1.26 >= 1.21'
++ bc -l
+ '[' 1 -eq 1 ']'
+ return 0
+ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-k121.yml ']'
+ '[' '!' -z '' -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-oc.yml ']'
+ '[' 0 = 1 -a -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150-eks.yml ']'
+ kubectl_bin get -o yaml statefulset/some-name-proxysql
++ mktemp
+ yq eval '
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "CLUSTER_HASH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "BACKUP_PATH")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "S3_BUCKET_URL")) |
			del(.spec.template.spec.containers[].env[] | select(.name == "AZURE_CONTAINER_NAME")) |
			del(.metadata.selfLink) |
			del(.metadata.deletionTimestamp) |
			del(.metadata.annotations."kubernetes.io/psp") |
			del(.metadata.annotations."batch.kubernetes.io/job-tracking") |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."k8s.v1.cni.cncf.io*") |
			del(.metadata.annotations."k8s.ovn.org/pod-networks") |
			del(.spec.template.metadata.annotations."last-applied-secret") |
			del(.. | select(has("batch.kubernetes.io/controller-uid"))."batch.kubernetes.io/controller-uid") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.spec.nodeName) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/env-secret-config-hash"))."percona.com/env-secret-config-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.. | select(has("imagePullSecrets")).imagePullSecrets) |
			del(.. | select(has("enableServiceLinks")).enableServiceLinks) |
			del(.status) |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.metadata.ownerReferences[].apiVersion) |
			del(.. | select(has("controller-uid")).controller-uid) |
			del(.. | select(has("preemptionPolicy")).preemptionPolicy) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "policy/v1beta1")) = "policy/v1" |
			del(.. | select(has("kubernetes.io/hostname"))."kubernetes.io/hostname") |
			(.. | select(tag == "!!str")) |= sub("upgrade-consistency-9051", "namespace") |
			(.. | select(tag == "!!str")) |= sub("kube-api-access-.*", "kube-api-access") |
			del(.. | select(has("annotations")).annotations | select(length==0)) |
			del(.. | select(.[] == "percona-xtradb-cluster-operator-workload-token*"))' -
+ local LAST_OUT=/tmp/tmp.X3CZ4WGgbG
++ mktemp
+ local LAST_ERR=/tmp/tmp.aCD0en5QqO
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get -o yaml statefulset/some-name-proxysql
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.X3CZ4WGgbG
+ cat /tmp/tmp.aCD0en5QqO
+ rm /tmp/tmp.X3CZ4WGgbG /tmp/tmp.aCD0en5QqO
+ return 0
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-1746/e2e-tests/upgrade-consistency/compare/statefulset_some-name-proxysql-1150.yml /tmp/tmp.NNcSZUPFDP/statefulset_some-name-proxysql.yml
+ destroy upgrade-consistency-9051
+ local namespace=upgrade-consistency-9051
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
+ grep -v level=info
+ tee /tmp/tmp.NNcSZUPFDP/operator.log
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+ grep -v 'the object has been modified'
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bMNnrovsz3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RoOpKliBIw
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bMNnrovsz3
++ cat /tmp/tmp.RoOpKliBIw
++ rm /tmp/tmp.bMNnrovsz3 /tmp/tmp.RoOpKliBIw
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-5bfd85498-fjkc8
++ mktemp
+ local LAST_OUT=/tmp/tmp.Bkc9W9G3El
++ mktemp
+ local LAST_ERR=/tmp/tmp.D4IL2wJB79
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-5bfd85498-fjkc8
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Bkc9W9G3El
+ cat /tmp/tmp.D4IL2wJB79
+ rm /tmp/tmp.Bkc9W9G3El /tmp/tmp.D4IL2wJB79
+ return 0
2024-07-01T03:25:27.946Z	INFO	setup	Manager starting up	{"gitCommit": "ef1f34c2341c8d03300153a4423cdf6baa839c53", "gitBranch": "PR-1746-ef1f34c2", "buildTime": "2024-07-01T01:45:38Z", "goVersion": "go1.22.4", "os": "linux", "arch": "amd64"}
2024-07-01T03:25:27.946Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.26.15-gke.1469000"}
2024-07-01T03:25:28.015Z	INFO	setup	Registering Components.
2024-07-01T03:25:29.722Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2024-07-01T03:25:29.726Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2024-07-01T03:25:29.726Z	INFO	controller-runtime.metrics	Starting metrics server
2024-07-01T03:25:29.726Z	INFO	controller-runtime.webhook	Starting webhook server
2024-07-01T03:25:29.726Z	INFO	setup	Starting the Cmd.
2024-07-01T03:25:29.726Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2024-07-01T03:25:29.727Z	INFO	controller-runtime.certwatcher	Starting certificate watcher
2024-07-01T03:25:29.727Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate
2024-07-01T03:25:29.727Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2024-07-01T03:25:29.827Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2024-07-01T03:25:29.846Z	DEBUG	events	percona-xtradb-cluster-operator-5bfd85498-fjkc8_79c5a4e7-c9b6-45b7-a998-f64d30401b34 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"78ccf7e6-e5bd-462a-9986-7d643df21d5b","apiVersion":"coordination.k8s.io/v1","resourceVersion":"53105"}, "reason": "LeaderElection"}
2024-07-01T03:25:29.846Z	INFO	Starting Controller	{"controller": "pxcbackup-controller"}
2024-07-01T03:25:29.846Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2024-07-01T03:25:29.846Z	INFO	Starting Controller	{"controller": "pxcrestore-controller"}
2024-07-01T03:25:29.846Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2024-07-01T03:25:29.846Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2024-07-01T03:25:29.846Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2024-07-01T03:25:29.846Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2024-07-01T03:25:29.952Z	INFO	Starting workers	{"controller": "pxcbackup-controller", "worker count": 1}
2024-07-01T03:25:29.952Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2024-07-01T03:25:30.059Z	INFO	Starting workers	{"controller": "pxcrestore-controller", "worker count": 1}
2024-07-01T03:26:08.753Z	INFO	User secrets updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "294e85e6-91a9-41cf-a251-f14b68728176", "secrets": "my-cluster-secrets"}
2024-07-01T03:27:27.787Z	INFO	User monitor: granted privileges	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7a928c27-d2f1-4606-b51e-b8b397526009"}
2024-07-01T03:27:27.819Z	INFO	monitor user privileges granted	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7a928c27-d2f1-4606-b51e-b8b397526009"}
2024-07-01T03:27:27.862Z	INFO	User xtrabackup: granted privileges	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7a928c27-d2f1-4606-b51e-b8b397526009"}
2024-07-01T03:27:27.960Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7a928c27-d2f1-4606-b51e-b8b397526009", "err": "get primary pxc pod: not found"}
2024-07-01T03:27:32.811Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7fc67ad3-0bdc-45a2-b730-5f1409a343a4", "err": "get primary pxc pod: not found"}
2024-07-01T03:27:38.033Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "69562c40-dec1-44f4-981c-de0b7a8bcc4d", "err": "get primary pxc pod: not found"}
2024-07-01T03:27:43.216Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "1d23df1c-4a01-4a12-9834-c6768cfc8825", "err": "get primary pxc pod: not found"}
2024-07-01T03:29:56.678Z	INFO	update PXC version (fetched from db)	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "3947b125-5aa1-4811-99d6-1c65adf88a27", "new version": "8.0.36-28.1"}
2024-07-01T03:29:59.869Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "3947b125-5aa1-4811-99d6-1c65adf88a27"}
2024-07-01T03:30:04.947Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "f31c9fb1-c7b8-448f-9475-eca75f1408b6"}
2024-07-01T03:30:10.237Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "648e54d4-15d1-4be7-bd56-cedd07a70137"}
2024-07-01T03:30:15.681Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "c22e09af-e67a-4bdd-9f49-e2fbb0c118b1"}
2024-07-01T03:30:21.020Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "6512d359-bf27-4605-a1f8-8d1238a4c96d"}
2024-07-01T03:30:26.343Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "dbd72d28-bd13-447e-b3e9-92f6db1166b6"}
2024-07-01T03:30:26.635Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "user": "root"}
2024-07-01T03:30:26.676Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "user": "operator"}
2024-07-01T03:30:26.729Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "user": "monitor"}
2024-07-01T03:30:26.768Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "user": "xtrabackup"}
2024-07-01T03:30:26.806Z	INFO	Password expiration policy updated	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "user": "replication"}
2024-07-01T03:30:28.612Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "7af11acb-5888-4bfd-84ec-c6422e7cbcb2", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-01T03:31:27.159Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "c8616e67-dc3c-4a2d-aafc-ce8b9cb8ff62", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:31:32.134Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "accb70a5-80a0-4f3c-a649-7266213dd5f1", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:31:37.368Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "9945094f-5aff-4de7-93d1-6e9b5e3a5cbf", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:32:20.404Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "5f21bbe2-6200-4de5-822e-bc1a5a652252", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-0: dial tcp 10.77.17.50:33062: connect: connection refused"}
2024-07-01T03:32:25.607Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "cf18a4d7-445f-4405-ab6e-d841d03fbe75", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:32:31.039Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "e900f3af-0b39-404e-a04e-a5d14b9336d5", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:32:36.231Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "980e07c4-c728-40a6-9623-dacea66e121c", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:32:41.489Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "622e22eb-ba67-41f8-9245-c25884b88e4b", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:32:46.683Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "d48e8c77-5c0f-4eb1-a77d-2ba9e7ce4837", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:32:51.884Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "4cb229e9-90ae-40bd-ac03-9961cae20564", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:33:00.841Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "1f8908a0-b439-4cfc-893d-6b342ce71e0c"}
2024-07-01T03:33:06.139Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "de2d3f94-fdf1-46c7-b7bd-084bb5b4c218"}
2024-07-01T03:33:11.329Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "a32adf45-3ae7-41d2-b1d2-eeafd06820cc"}
2024-07-01T03:33:15.766Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "4b7bdd7b-add5-4888-8639-a51e7cbbfeb6", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-01T03:34:08.848Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "9084f13b-a782-400f-8d6c-33ca3fd0d503", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:34:13.796Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "086fe16c-e951-48ae-9037-aab7b67c7123", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:34:19.137Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "77b618ba-7e0d-4879-b112-685398cef440", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:35:22.227Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "9fc0ae55-489f-49a8-97c5-7b7b8bf77a59", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.77.17.53:33062: i/o timeout"}
2024-07-01T03:35:27.401Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "4bcbc696-6bd7-49e0-a776-a2022ad8e377", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:35:32.594Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "ab25f834-5c00-4dc2-9fcd-bea51b765f1f", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:35:37.792Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "03323bcb-c940-4424-945e-068f73d064fc", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:35:47.563Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "749bda72-d0b5-4096-842f-5458ec56c7e7"}
2024-07-01T03:35:51.744Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "f2f4b935-0606-4571-9744-793e4db09f3e"}
2024-07-01T03:35:57.245Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "73254fa6-521a-4277-97fd-6db66fc1ec4d"}
2024-07-01T03:35:59.525Z	ERROR	sync users	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "8ed8876a-368d-490e-902d-97b8dcd5dd28", "error": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / ", "errorVerbose": "exec syncusers: unable to upgrade connection: container not found (\"proxysql\") /  / \ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).syncPXCUsersWithProxySQL\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/users.go:920\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1246\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1695"}
2024-07-01T03:36:30.901Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "5e3fd5d3-d060-4652-ae7f-3c2fab4364fe", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-2: dial tcp: lookup some-name-pxc-2.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:36:57.085Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "4c23a871-eac4-4c24-9f04-d1cf34cd3111", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:37:02.406Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "88b944b5-ae7a-46c5-aa70-182f91857743", "err": "failed to ensure cluster readonly status: connect to pod some-name-pxc-1: dial tcp: lookup some-name-pxc-1.some-name-pxc.upgrade-consistency-9051 on 10.156.96.10:53: no such host"}
2024-07-01T03:37:49.987Z	INFO	reconcile replication error	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "ff0b7006-0423-420e-87ab-f217d486da4f", "err": "failed to connect to pod some-name-pxc-0: dial tcp 10.77.17.55:33062: connect: connection refused"}
2024-07-01T03:37:55.202Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "b03a0aed-5432-4699-8ff6-4fddb9795f25", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:38:00.597Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "d1025326-b3a9-414e-ae20-cbd25b319388", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:38:05.845Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "5956b17c-cab2-49c1-8de7-adc9ffe62717", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:38:16.444Z	INFO	Unable to find primary pod for replication. No pod with name or ip like this	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "979a173a-9505-4b61-8ffe-ee16ffd6ae1f", "primary name": "some-name-pxc-0.some-name-pxc.upgrade-consistency-9051.svc.cluster.local"}
2024-07-01T03:38:30.800Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "88939b75-0c44-45c0-bce0-495f637ec248"}
2024-07-01T03:38:35.824Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "539ce8e3-1b65-4b31-90d9-9a1e786f12ee"}
2024-07-01T03:38:41.162Z	DEBUG	PXC users synced with ProxySQL	{"controller": "pxc-controller", "namespace": "upgrade-consistency-9051", "name": "some-name", "reconcileID": "3aafffcc-6eb4-4328-b2fd-fd721b90d310"}
github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).resyncPXCUsersWithProxySQL.func1
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:222
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:261
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.18.4/pkg/internal/controller/controller.go:324
	/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:1248
[mysql] 2024/07/01 03:32:20 connection.go:49: read tcp 10.77.17.48:32874->10.77.17.50:33062: read: connection reset by peer
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n upgrade-consistency-9051 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.1q9B03BQny
++ mktemp
+ local LAST_ERR=/tmp/tmp.Hz1LW8Y7I7
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1q9B03BQny
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.Hz1LW8Y7I7
+ rm /tmp/tmp.1q9B03BQny /tmp/tmp.Hz1LW8Y7I7
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.ADfKaCfjiI
++ mktemp
+ local LAST_ERR=/tmp/tmp.PhBTVXU7NW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ADfKaCfjiI
No resources found
+ cat /tmp/tmp.PhBTVXU7NW
+ rm /tmp/tmp.ADfKaCfjiI /tmp/tmp.PhBTVXU7NW
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.NY5vARpXX4
++ mktemp
+ local LAST_ERR=/tmp/tmp.9lueJEBAHV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.NY5vARpXX4
No resources found
+ cat /tmp/tmp.9lueJEBAHV
+ rm /tmp/tmp.NY5vARpXX4 /tmp/tmp.9lueJEBAHV
+ return 0
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.uMMetoPsaK
++ mktemp
+ local LAST_ERR=/tmp/tmp.pGT9F6W4yb
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uMMetoPsaK
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.pGT9F6W4yb
+ rm /tmp/tmp.uMMetoPsaK /tmp/tmp.pGT9F6W4yb
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.14.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace upgrade-consistency-9051
+ rm -rf /tmp/tmp.NNcSZUPFDP
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.el4ObTWyoE
+ desc 'test passed'
+ set +o xtrace
++ mktemp


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

+ local LAST_OUT=/tmp/tmp.vJNro4vxGk
++ mktemp
+ local LAST_ERR=/tmp/tmp.DRvmraMoJt
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.YkjJAiNgNK
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace upgrade-consistency-9051