Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/logs/upgrade-consistency-8-0.log
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
WARNING: version difference between client (1.32) and server (1.28) exceeds the supported minor version skew of +/-1
+ cluster=some-name
+ main
+ create_infra upgrade-consistency-31638
+ local ns=upgrade-consistency-31638
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n upgrade-consistency-18917 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.5xAm5aSmv2
++ mktemp
+ local LAST_ERR=/tmp/tmp.w51XNFShjk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5xAm5aSmv2
perconaxtradbcluster.pxc.percona.com "some-name" deleted
+ cat /tmp/tmp.w51XNFShjk
+ rm /tmp/tmp.5xAm5aSmv2 /tmp/tmp.w51XNFShjk
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.C4zCIFOqwS
++ mktemp
+ local LAST_ERR=/tmp/tmp.b4OXE3Qfuk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.C4zCIFOqwS
No resources found
+ cat /tmp/tmp.b4OXE3Qfuk
+ rm /tmp/tmp.C4zCIFOqwS /tmp/tmp.b4OXE3Qfuk
+ return 0
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.eMgew42081
++ mktemp
+ local LAST_ERR=/tmp/tmp.OerGJGOhNV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.eMgew42081
No resources found
+ cat /tmp/tmp.OerGJGOhNV
+ rm /tmp/tmp.eMgew42081 /tmp/tmp.OerGJGOhNV
+ return 0
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrolebinding
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ kubectl get clusterrole
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.CIuVOJ4nFv
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ kubectl_bin get ns
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.2njjBvQ6Eb
+ awk '{print$1}'
+ local LAST_ERR=/tmp/tmp.onbyRHTWWM
+ local exit_status=0
++ seq 0 2
+ xargs kubectl delete ns
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ mktemp
+ local LAST_ERR=/tmp/tmp.8xidg4oLrc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2njjBvQ6Eb
+ cat /tmp/tmp.8xidg4oLrc
+ rm /tmp/tmp.2njjBvQ6Eb /tmp/tmp.8xidg4oLrc
+ return 0
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
namespace "upgrade-consistency-18917" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CIuVOJ4nFv
namespace "pxc-operator" deleted
+ cat /tmp/tmp.onbyRHTWWM
+ rm /tmp/tmp.CIuVOJ4nFv /tmp/tmp.onbyRHTWWM
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.k5SnuNowBn
++ mktemp
+ local LAST_ERR=/tmp/tmp.d6AKNOK0RB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.k5SnuNowBn
namespace/pxc-operator created
+ cat /tmp/tmp.d6AKNOK0RB
+ rm /tmp/tmp.k5SnuNowBn /tmp/tmp.d6AKNOK0RB
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.eJzNipyHUf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.pRoOR4JALM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.eJzNipyHUf
++ cat /tmp/tmp.pRoOR4JALM
++ rm /tmp/tmp.eJzNipyHUf /tmp/tmp.pRoOR4JALM
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.dyHvz8ouFm
++ mktemp
+ local LAST_ERR=/tmp/tmp.MOBbOA2VC0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.dyHvz8ouFm
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2" modified.
+ cat /tmp/tmp.MOBbOA2VC0
+ rm /tmp/tmp.dyHvz8ouFm /tmp/tmp.MOBbOA2VC0
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Pe7TwTp8c
++ mktemp
+ local LAST_ERR=/tmp/tmp.tD1fWHrgZC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9Pe7TwTp8c
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.tD1fWHrgZC
+ rm /tmp/tmp.9Pe7TwTp8c /tmp/tmp.tD1fWHrgZC
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.6ZOHJQLJ7T
++ mktemp
+ local LAST_ERR=/tmp/tmp.Rp2q19DJrN
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6ZOHJQLJ7T
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.Rp2q19DJrN
+ rm /tmp/tmp.6ZOHJQLJ7T /tmp/tmp.Rp2q19DJrN
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "DEBUG"' -
+ kubectl_bin apply -f -
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nOv5k21tQC
++ mktemp
+ local LAST_ERR=/tmp/tmp.SW5nPsSHQ6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nOv5k21tQC
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.SW5nPsSHQ6
+ rm /tmp/tmp.nOv5k21tQC /tmp/tmp.SW5nPsSHQ6
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.Et4mVRfdVK
++ mktemp
+ local LAST_ERR=/tmp/tmp.zupTQIzHIJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Et4mVRfdVK
pod/percona-xtradb-cluster-operator-67cbddd7f9-t2wzp condition met
+ cat /tmp/tmp.zupTQIzHIJ
+ rm /tmp/tmp.Et4mVRfdVK /tmp/tmp.zupTQIzHIJ
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AmOcS6Mcp7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iMFiTcqP0S
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AmOcS6Mcp7
++ cat /tmp/tmp.iMFiTcqP0S
++ rm /tmp/tmp.AmOcS6Mcp7 /tmp/tmp.iMFiTcqP0S
++ return 0
+ wait_pod percona-xtradb-cluster-operator-67cbddd7f9-t2wzp 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-67cbddd7f9-t2wzp
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-67cbddd7f9-t2wzp
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-67cbddd7f9-t2wzp condition met
waiting for pod/percona-xtradb-cluster-operator-67cbddd7f9-t2wzp to become Ready.Ok
+ sleep 3
+ create_namespace upgrade-consistency-31638
+ local namespace=upgrade-consistency-31638
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ egrep -v '^kube-|^default$|Terminating|pxc-operator|openshift|^NAME'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces upgrade-consistency-31638'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces upgrade-consistency-31638
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace upgrade-consistency-31638
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.lPiocS9SUV
++ mktemp
+ local LAST_ERR=/tmp/tmp.qoD7AmCJdR
+ local exit_status=0
++ seq 0 2
++ mktemp
+ local LAST_OUT=/tmp/tmp.KdvADNUVqd
++ mktemp
+ local LAST_ERR=/tmp/tmp.fXT46X0mKJ
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-31638
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-31638
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.KdvADNUVqd
+ cat /tmp/tmp.fXT46X0mKJ
+ rm /tmp/tmp.KdvADNUVqd /tmp/tmp.fXT46X0mKJ
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace upgrade-consistency-31638
namespace "gke-managed-system" deleted
namespace "gmp-public" deleted
namespace "gmp-system" deleted
Error from server (Forbidden): namespaces "default" is forbidden: this namespace may not be deleted
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.lPiocS9SUV
+ cat /tmp/tmp.qoD7AmCJdR
Error from server (NotFound): namespaces "upgrade-consistency-31638" not found
+ rm /tmp/tmp.lPiocS9SUV /tmp/tmp.qoD7AmCJdR
+ return 1
+ :
+ wait_for_delete namespace/upgrade-consistency-31638
+ local res=namespace/upgrade-consistency-31638
+ echo -n 'waiting for namespace/upgrade-consistency-31638 to be deleted'
waiting for namespace/upgrade-consistency-31638 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "upgrade-consistency-31638" not found
+ desc 'create namespace upgrade-consistency-31638'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace upgrade-consistency-31638
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace upgrade-consistency-31638
++ mktemp
+ local LAST_OUT=/tmp/tmp.8oMQ0jCKws
++ mktemp
+ local LAST_ERR=/tmp/tmp.awdEuPhTUB
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace upgrade-consistency-31638
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.8oMQ0jCKws
namespace/upgrade-consistency-31638 created
+ cat /tmp/tmp.awdEuPhTUB
+ rm /tmp/tmp.8oMQ0jCKws /tmp/tmp.awdEuPhTUB
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kkhU03VfrR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BTH0AcicIP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kkhU03VfrR
++ cat /tmp/tmp.BTH0AcicIP
++ rm /tmp/tmp.kkhU03VfrR /tmp/tmp.BTH0AcicIP
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2 --namespace=upgrade-consistency-31638
++ mktemp
+ local LAST_OUT=/tmp/tmp.xHK6FGcWkp
++ mktemp
+ local LAST_ERR=/tmp/tmp.9PJUQx9QwX
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2 --namespace=upgrade-consistency-31638
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xHK6FGcWkp
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-1697-26afa527-8-cluster2" modified.
+ cat /tmp/tmp.9PJUQx9QwX
+ rm /tmp/tmp.xHK6FGcWkp /tmp/tmp.9PJUQx9QwX
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.RenRqOEnR2
++ mktemp
+ local LAST_ERR=/tmp/tmp.br5oJvHklM
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RenRqOEnR2
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.br5oJvHklM
+ rm /tmp/tmp.RenRqOEnR2 /tmp/tmp.br5oJvHklM
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-consistency-31638~
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_OUT=/tmp/tmp.zF17bWW5dV
++ mktemp
+ local LAST_ERR=/tmp/tmp.F7FkO2xoy4
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zF17bWW5dV
deployment.apps/pxc-client created
+ cat /tmp/tmp.F7FkO2xoy4
+ rm /tmp/tmp.zF17bWW5dV /tmp/tmp.F7FkO2xoy4
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/upgrade-consistency/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2BrlMGc5vL
++ mktemp
+ local LAST_ERR=/tmp/tmp.AK1aTQZHq3
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/upgrade-consistency/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2BrlMGc5vL
secret/my-cluster-secrets created
+ cat /tmp/tmp.AK1aTQZHq3
+ rm /tmp/tmp.2BrlMGc5vL /tmp/tmp.AK1aTQZHq3
+ return 0
+ yq eval '.spec.crVersion = "1.14.1"' -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ kubectl_bin apply -f -
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.upgrade-consistency-31638~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-1697-26afa527#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1697/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.gLDK4ngNus
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ local LAST_ERR=/tmp/tmp.Rt2l3ufl1T
+ local exit_status=0
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.gLDK4ngNus
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.Rt2l3ufl1T
+ rm /tmp/tmp.gLDK4ngNus /tmp/tmp.Rt2l3ufl1T
+ return 0
+ desc 'test 1.14.1'
+ set +o xtrace


-----------------------------------------------------------------------------------
test 1.14.1
-----------------------------------------------------------------------------------

+ kubectl_bin patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.14.1"}
    }'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vV0GZGIE6a
++ mktemp
+ local LAST_ERR=/tmp/tmp.XCbTms44Bo
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc some-name --type=merge --patch '{
        "spec": {"crVersion":"1.14.1"}
    }'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.vV0GZGIE6a
perconaxtradbcluster.pxc.percona.com/some-name patched (no change)
+ cat /tmp/tmp.XCbTms44Bo
+ rm /tmp/tmp.vV0GZGIE6a /tmp/tmp.XCbTms44Bo
+ return 0
+ wait_cluster_consistency some-name 3 2
+ local cluster_name=some-name
+ local cluster_size=3
+ local proxy_size=2
+ '[' -z 2 ']'
+ desc 'wait cluster consistency'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait cluster consistency
-----------------------------------------------------------------------------------

+ local i=0
+ local max=36
+ sleep 7
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.JwwXVQeFfW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RdAWOyEdfW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.JwwXVQeFfW
++ cat /tmp/tmp.RdAWOyEdfW
++ rm /tmp/tmp.JwwXVQeFfW /tmp/tmp.RdAWOyEdfW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 0 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cJD64l35jd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6yb6m1HarL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cJD64l35jd
++ cat /tmp/tmp.6yb6m1HarL
++ rm /tmp/tmp.cJD64l35jd /tmp/tmp.6yb6m1HarL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 1 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8iBCg8dVB6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3E6JOVxMeJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.8iBCg8dVB6
++ cat /tmp/tmp.3E6JOVxMeJ
++ rm /tmp/tmp.8iBCg8dVB6 /tmp/tmp.3E6JOVxMeJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 2 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.yLW6bCRbwc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.cDD08xJrTJ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.yLW6bCRbwc
++ cat /tmp/tmp.cDD08xJrTJ
++ rm /tmp/tmp.yLW6bCRbwc /tmp/tmp.cDD08xJrTJ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 3 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NVPjF57kUf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6uaEncdzJY
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.NVPjF57kUf
++ cat /tmp/tmp.6uaEncdzJY
++ rm /tmp/tmp.NVPjF57kUf /tmp/tmp.6uaEncdzJY
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 4 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oPcc9I5nLy
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BalUpOgycs
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.oPcc9I5nLy
++ cat /tmp/tmp.BalUpOgycs
++ rm /tmp/tmp.oPcc9I5nLy /tmp/tmp.BalUpOgycs
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 5 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kxSp9PayHi
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jdDFZnpQVg
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kxSp9PayHi
++ cat /tmp/tmp.jdDFZnpQVg
++ rm /tmp/tmp.kxSp9PayHi /tmp/tmp.jdDFZnpQVg
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 6 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EL3iVil6ld
+++ mktemp
++ local LAST_ERR=/tmp/tmp.swDOddhaaZ
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.EL3iVil6ld
++ cat /tmp/tmp.swDOddhaaZ
++ rm /tmp/tmp.EL3iVil6ld /tmp/tmp.swDOddhaaZ
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 7 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uCpRx49I9i
+++ mktemp
++ local LAST_ERR=/tmp/tmp.IUxiAW2pb2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.uCpRx49I9i
++ cat /tmp/tmp.IUxiAW2pb2
++ rm /tmp/tmp.uCpRx49I9i /tmp/tmp.IUxiAW2pb2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 8 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.KpbXL8rJL3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UhHJebqzOf
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.KpbXL8rJL3
++ cat /tmp/tmp.UhHJebqzOf
++ rm /tmp/tmp.KpbXL8rJL3 /tmp/tmp.UhHJebqzOf
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 9 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nB64eXEDFe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FRipA4Ku0A
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nB64eXEDFe
++ cat /tmp/tmp.FRipA4Ku0A
++ rm /tmp/tmp.nB64eXEDFe /tmp/tmp.FRipA4Ku0A
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 10 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vmKNLc41sA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZeCRsfKwHu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vmKNLc41sA
++ cat /tmp/tmp.ZeCRsfKwHu
++ rm /tmp/tmp.vmKNLc41sA /tmp/tmp.ZeCRsfKwHu
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 11 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jyk3WiGdKl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gR0MtGKpjX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jyk3WiGdKl
++ cat /tmp/tmp.gR0MtGKpjX
++ rm /tmp/tmp.jyk3WiGdKl /tmp/tmp.gR0MtGKpjX
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 12 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1jFTtnW2kk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zSkARzhgfi
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.1jFTtnW2kk
++ cat /tmp/tmp.zSkARzhgfi
++ rm /tmp/tmp.1jFTtnW2kk /tmp/tmp.zSkARzhgfi
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 13 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7BRHuvMDvt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rocGUD5fi4
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.7BRHuvMDvt
++ cat /tmp/tmp.rocGUD5fi4
++ rm /tmp/tmp.7BRHuvMDvt /tmp/tmp.rocGUD5fi4
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 14 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ejbkl4Saax
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ST3wvFIY1g
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Ejbkl4Saax
++ cat /tmp/tmp.ST3wvFIY1g
++ rm /tmp/tmp.Ejbkl4Saax /tmp/tmp.ST3wvFIY1g
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 15 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lITW2HMsg9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rNmNHlCTv7
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lITW2HMsg9
++ cat /tmp/tmp.rNmNHlCTv7
++ rm /tmp/tmp.lITW2HMsg9 /tmp/tmp.rNmNHlCTv7
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 16 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wsUgQJwFOB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UXVozyyNBW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wsUgQJwFOB
++ cat /tmp/tmp.UXVozyyNBW
++ rm /tmp/tmp.wsUgQJwFOB /tmp/tmp.UXVozyyNBW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 17 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.kS0mNTcKFJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.78V2jFO4Wb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.kS0mNTcKFJ
++ cat /tmp/tmp.78V2jFO4Wb
++ rm /tmp/tmp.kS0mNTcKFJ /tmp/tmp.78V2jFO4Wb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 18 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.cP31v1PiYq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oG2wj6zJ8I
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.cP31v1PiYq
++ cat /tmp/tmp.oG2wj6zJ8I
++ rm /tmp/tmp.cP31v1PiYq /tmp/tmp.oG2wj6zJ8I
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 19 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AlTUpYtAU3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7xPYMFUQn2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.AlTUpYtAU3
++ cat /tmp/tmp.7xPYMFUQn2
++ rm /tmp/tmp.AlTUpYtAU3 /tmp/tmp.7xPYMFUQn2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 20 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tjgFmVt2YI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XPjm9t8B7a
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.tjgFmVt2YI
++ cat /tmp/tmp.XPjm9t8B7a
++ rm /tmp/tmp.tjgFmVt2YI /tmp/tmp.XPjm9t8B7a
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 21 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9O2oQP9hEE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LjgOu7MPyW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.9O2oQP9hEE
++ cat /tmp/tmp.LjgOu7MPyW
++ rm /tmp/tmp.9O2oQP9hEE /tmp/tmp.LjgOu7MPyW
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 22 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3tTGbvEvgc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.y1LwEtLpIM
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3tTGbvEvgc
++ cat /tmp/tmp.y1LwEtLpIM
++ rm /tmp/tmp.3tTGbvEvgc /tmp/tmp.y1LwEtLpIM
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 23 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0ze8y6Tfn8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.54jxmBKNIA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0ze8y6Tfn8
++ cat /tmp/tmp.54jxmBKNIA
++ rm /tmp/tmp.0ze8y6Tfn8 /tmp/tmp.54jxmBKNIA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 24 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dlAmuubeIb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J7E8v7BQ6U
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dlAmuubeIb
++ cat /tmp/tmp.J7E8v7BQ6U
++ rm /tmp/tmp.dlAmuubeIb /tmp/tmp.J7E8v7BQ6U
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 25 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vENT6RTbWq
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lnfHp9rlE2
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vENT6RTbWq
++ cat /tmp/tmp.lnfHp9rlE2
++ rm /tmp/tmp.vENT6RTbWq /tmp/tmp.lnfHp9rlE2
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 26 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.E1BUohH7Lk
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ift1fI3Zfb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.E1BUohH7Lk
++ cat /tmp/tmp.Ift1fI3Zfb
++ rm /tmp/tmp.E1BUohH7Lk /tmp/tmp.Ift1fI3Zfb
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 27 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vr9oZSUPXb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fwm41E1Sai
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.vr9oZSUPXb
++ cat /tmp/tmp.fwm41E1Sai
++ rm /tmp/tmp.vr9oZSUPXb /tmp/tmp.fwm41E1Sai
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 28 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xtD5QRMTUZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VfzO9dUkYR
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.xtD5QRMTUZ
++ cat /tmp/tmp.VfzO9dUkYR
++ rm /tmp/tmp.xtD5QRMTUZ /tmp/tmp.VfzO9dUkYR
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 29 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jUfk88XwRM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gx4K4fiTvx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.jUfk88XwRM
++ cat /tmp/tmp.gx4K4fiTvx
++ rm /tmp/tmp.jUfk88XwRM /tmp/tmp.gx4K4fiTvx
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 30 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ETMTC9qPxl
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BVWdffl9TS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ETMTC9qPxl
++ cat /tmp/tmp.BVWdffl9TS
++ rm /tmp/tmp.ETMTC9qPxl /tmp/tmp.BVWdffl9TS
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 31 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gFDkBiw68M
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j7zw0rZ7Mh
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.gFDkBiw68M
++ cat /tmp/tmp.j7zw0rZ7Mh
++ rm /tmp/tmp.gFDkBiw68M /tmp/tmp.j7zw0rZ7Mh
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 32 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.PdXGfS3STs
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RDxISV34XL
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.PdXGfS3STs
++ cat /tmp/tmp.RDxISV34XL
++ rm /tmp/tmp.PdXGfS3STs /tmp/tmp.RDxISV34XL
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 33 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.30xTwvPf5b
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JTDxUrdWeA
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.30xTwvPf5b
++ cat /tmp/tmp.JTDxUrdWeA
++ rm /tmp/tmp.30xTwvPf5b /tmp/tmp.JTDxUrdWeA
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 34 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sjdfMdJiOT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.x8rrXFE2dc
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.sjdfMdJiOT
++ cat /tmp/tmp.x8rrXFE2dc
++ rm /tmp/tmp.sjdfMdJiOT /tmp/tmp.x8rrXFE2dc
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 35 -ge 36 ]]
+ let i+=1
++ kubectl_bin get pxc some-name -o 'jsonpath={.status.state}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.4ZfCFT7o1K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mlosgVEEqN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pxc some-name -o 'jsonpath={.status.state}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.4ZfCFT7o1K
++ cat /tmp/tmp.mlosgVEEqN
++ rm /tmp/tmp.4ZfCFT7o1K /tmp/tmp.mlosgVEEqN
++ return 0
+ [[ initializing == \r\e\a\d\y ]]
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 20
+ [[ 36 -ge 36 ]]
+ echo 'Something went wrong waiting for cluster consistency!'
Something went wrong waiting for cluster consistency!
+ exit 1