++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/upgrade.log'
Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/upgrade.log
++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP=
++ oc get projects
error: the server doesn't have a resource type "projects"
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
+++ kubectl version -o json
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+++ helm version -c
+++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/'
++ HELM_VERSION=v3.14.4
++ '[' v3 == v2 ']'
+ main
+ create_namespace upgrade-24467
+ local namespace=upgrade-24467
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ cleanup_rbac
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ xargs kubectl delete ns
++ mktemp
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.9NXzA5Q2IY
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.AwYgsjJlz6
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ local LAST_OUT=/tmp/tmp.vhBJHWh1nn
++ seq 0 2
++ mktemp
+ local LAST_ERR=/tmp/tmp.xTYMDRGH5e
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.9NXzA5Q2IY
+ cat /tmp/tmp.AwYgsjJlz6
E0423 13:33:13.627636   23833 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:13.979635   23833 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:14.086482   23833 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:14.192935   23833 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
+ rm /tmp/tmp.9NXzA5Q2IY /tmp/tmp.AwYgsjJlz6
+ return 0
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
E0423 13:33:15.370009   23971 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:15.586580   23971 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:15.694589   23971 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:15.802874   23971 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.vhBJHWh1nn
+ cat /tmp/tmp.xTYMDRGH5e
E0423 13:33:21.578566   24275 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:21.789986   24275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:21.897442   24275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:22.004645   24275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.vhBJHWh1nn /tmp/tmp.xTYMDRGH5e
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
++ mktemp
+ local LAST_OUT=/tmp/tmp.Osk7yhCnn8
++ mktemp
+ local LAST_ERR=/tmp/tmp.rc8b3cN8yE
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.Osk7yhCnn8
+ cat /tmp/tmp.rc8b3cN8yE
E0423 13:33:39.652855   25202 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:39.762461   25202 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:39.869201   25202 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:39.976197   25202 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.Osk7yhCnn8 /tmp/tmp.rc8b3cN8yE
+ return 1
+ true
+ kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth
E0423 13:33:49.667172   25662 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:49.779132   25662 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:49.886615   25662 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0423 13:33:49.994206   25662 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found
+ true
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin delete namespace upgrade-24467
++ mktemp
+ local LAST_OUT=/tmp/tmp.qCh9O1hfF6
++ mktemp
+ local LAST_ERR=/tmp/tmp.TN2HaSFIHi
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete namespace upgrade-24467
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete namespace upgrade-24467
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete namespace upgrade-24467
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.qCh9O1hfF6
+ cat /tmp/tmp.TN2HaSFIHi
Error from server (NotFound): namespaces "upgrade-24467" not found
+ rm /tmp/tmp.qCh9O1hfF6 /tmp/tmp.TN2HaSFIHi
+ return 1
+ :
+ wait_for_delete namespace/upgrade-24467
+ local res=namespace/upgrade-24467
+ set +o xtrace
namespace/upgrade-24467 - Error from server (NotFound): namespaces "upgrade-24467" not found
+ kubectl_bin create namespace upgrade-24467
++ mktemp
+ local LAST_OUT=/tmp/tmp.IUP9UyZ08E
++ mktemp
+ local LAST_ERR=/tmp/tmp.uHHeugbuAU
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl create namespace upgrade-24467
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.IUP9UyZ08E
namespace/upgrade-24467 created
+ cat /tmp/tmp.uHHeugbuAU
+ rm /tmp/tmp.IUP9UyZ08E /tmp/tmp.uHHeugbuAU
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YgAoArVGl1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Rlfdv6sXZ6
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl config current-context
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.YgAoArVGl1
++ cat /tmp/tmp.Rlfdv6sXZ6
++ rm /tmp/tmp.YgAoArVGl1 /tmp/tmp.Rlfdv6sXZ6
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-upgrade --namespace=upgrade-24467
++ mktemp
+ local LAST_OUT=/tmp/tmp.6qzodOYjbD
++ mktemp
+ local LAST_ERR=/tmp/tmp.oKekVWTJkk
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-upgrade --namespace=upgrade-24467
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6qzodOYjbD
Context "gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-upgrade" modified.
+ cat /tmp/tmp.oKekVWTJkk
+ rm /tmp/tmp.6qzodOYjbD /tmp/tmp.oKekVWTJkk
+ return 0
+ cluster=some-name
+ operator_action=install
+ create_or_upgrade_to_version some-name 1.5.0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install
+ local cluster=some-name
+ local pgo_tag=1.5.0
+ local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=install
+ IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.0-postgres-operator
+ IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.0-pgo-deployer
+ IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer
+ IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha
+ IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger
+ [[ xinstall == \x\i\n\s\t\a\l\l ]]
+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local action=install
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install upgrade-24467 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=install
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-24467"#g'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-24467"#g'
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml 'data[values.yaml]'
++ cut -d: -f1
++ echo percona/percona-postgresql-operator:1.5.0
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g'
+ cat /dev/fd/63 -
++ echo 'crunchy_debug: "true"'
++ cut -d: -f2
++ echo percona/percona-postgresql-operator:1.5.0
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.0"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w -d3 - 'subjects[0].namespace' upgrade-24467
+ yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.0-pgo-deployer
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.4.0_operator.yaml metadata.namespace upgrade-24467
++ cat /tmp/tmp.irtas5OIUp/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-24467"
namespace_mode: "dynamic"
pgo_image_prefix: "percona/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "1.5.0"
pgo_operator_namespace: "upgrade-24467"'
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.J7zt3eJyl7
++ mktemp
+ local LAST_ERR=/tmp/tmp.F5gjD7UaPI
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.J7zt3eJyl7
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.F5gjD7UaPI
+ rm /tmp/tmp.J7zt3eJyl7 /tmp/tmp.F5gjD7UaPI
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.J7Vi1IwYer
++ mktemp
+ local LAST_ERR=/tmp/tmp.KabangSUJn
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.J7Vi1IwYer
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.KabangSUJn
+ rm /tmp/tmp.J7Vi1IwYer /tmp/tmp.KabangSUJn
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uOjifeLprb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sjTEEfxSHd
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.uOjifeLprb
++ cat /tmp/tmp.sjTEEfxSHd
++ rm /tmp/tmp.uOjifeLprb /tmp/tmp.sjTEEfxSHd
++ return 0
+ wait_pod postgres-operator-5975cfcfc5-gb9tt
+ local pod=postgres-operator-5975cfcfc5-gb9tt
+ local ns=
+ set +o xtrace
postgres-operator-5975cfcfc5-gb9tttrue
+ [[ -n '' ]]
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.d4Fk1RaTTS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Oie70zqVQO
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 0
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 4
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 8
++ set -o errexit
++ cat /tmp/tmp.d4Fk1RaTTS
++ cat /tmp/tmp.Oie70zqVQO
Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found
++ rm /tmp/tmp.d4Fk1RaTTS /tmp/tmp.Oie70zqVQO
++ return 1
+ [[ x == \x ]]
+ desc 'start cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
start cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name
+ local cluster=some-name
+ local namespace=upgrade-24467
+ yq w - metadata.name some-name-users-secret
+ yq w - stringData.some-name some-name_pass
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n upgrade-24467 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.dFXeC74xW1
++ mktemp
+ local LAST_ERR=/tmp/tmp.LHCfB0VuWs
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.dFXeC74xW1
secret/some-name-users-secret created
+ cat /tmp/tmp.LHCfB0VuWs
+ rm /tmp/tmp.dFXeC74xW1 /tmp/tmp.LHCfB0VuWs
+ return 0
+ kubectl_bin -n upgrade-24467 apply -f -
+ yq w - metadata.name some-name-pmm-secret
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ywaPfiW9eX
++ mktemp
+ local LAST_ERR=/tmp/tmp.ONWVqfn7dl
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ywaPfiW9eX
secret/some-name-pmm-secret created
+ cat /tmp/tmp.ONWVqfn7dl
+ rm /tmp/tmp.ywaPfiW9eX /tmp/tmp.ONWVqfn7dl
+ return 0
+ spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local backup=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ local namespace=upgrade-24467
+ local root_via_pgbouncer=
+ desc 'create fresh PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create fresh PG cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name upgrade-24467
+ local cluster=some-name
+ local namespace=upgrade-24467
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - metadata.name some-name-users-secret
+ yq w - stringData.some-name some-name_pass
+ kubectl_bin -n upgrade-24467 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.hUSDojpdyK
++ mktemp
+ local LAST_ERR=/tmp/tmp.eqUvChtUz4
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.hUSDojpdyK
secret/some-name-users-secret configured
+ cat /tmp/tmp.eqUvChtUz4
+ rm /tmp/tmp.hUSDojpdyK /tmp/tmp.eqUvChtUz4
+ return 0
+ yq w - metadata.name some-name-pmm-secret
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n upgrade-24467 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.V2Seru2c4D
++ mktemp
+ local LAST_ERR=/tmp/tmp.L5i5am5OuI
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.V2Seru2c4D
secret/some-name-pmm-secret configured
+ cat /tmp/tmp.L5i5am5OuI
+ rm /tmp/tmp.V2Seru2c4D /tmp/tmp.L5i5am5OuI
+ return 0
+ kubectl_bin -n upgrade-24467 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.O00CKzW6Ig
++ mktemp
+ local LAST_ERR=/tmp/tmp.SBReUIEFk1
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.O00CKzW6Ig
deployment.apps/pg-client created
+ cat /tmp/tmp.SBReUIEFk1
+ rm /tmp/tmp.O00CKzW6Ig /tmp/tmp.SBReUIEFk1
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_cr.yaml some-name false false false false false false false false
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local root_via_pgbouncer=
+ yq w - metadata.labels.deployment-name some-name
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.name some-name
+ yq w - spec.clustername some-name
+ yq w - spec.database some-name
+ yq w - spec.secretsName some-name-users-secret
+ yq w - spec.pgPrimary.image percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha
+ yq w - spec.user some-name
+ yq w - spec.keepData false
+ yq w - metadata.labels.name some-name
+ yq w - spec.backup.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - spec.backup.backrestRepoImage percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo
+ yq w - spec.pgBadger.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger
+ yq w - spec.name some-name
+ yq w - spec.pgBouncer.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer
+ yq w - spec.keepBackups false
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.irtas5OIUp/cr.yaml -n upgrade-24467
++ mktemp
+ local LAST_OUT=/tmp/tmp.KMdkQEEE3d
++ mktemp
+ local LAST_ERR=/tmp/tmp.5RDCXDd40x
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /tmp/tmp.irtas5OIUp/cr.yaml -n upgrade-24467
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.KMdkQEEE3d
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.5RDCXDd40x
+ rm /tmp/tmp.KMdkQEEE3d /tmp/tmp.5RDCXDd40x
+ return 0
+ wait_deployment some-name-backrest-shared-repo
+ local name=some-name-backrest-shared-repo
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name-backrest-shared-repo...
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name...........
+ [[ false == \g\c\s\+* ]]
+ wait_job_completion some-name-stanza-create false
+ local job=some-name-stanza-create
+ local mandatory=false
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
some-name-stanza-create+ retry=0
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name
+ local job=backrest-backup-some-name
+ local mandatory=true
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
backrest-backup-some-name+ retry=0
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UFiSabRFBv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.HM6nSsT7Ci
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.UFiSabRFBv
++ cat /tmp/tmp.HM6nSsT7Ci
++ rm /tmp/tmp.UFiSabRFBv /tmp/tmp.HM6nSsT7Ci
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9hy0eiQFNH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.o7PpafUqPS
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.9hy0eiQFNH
+++ cat /tmp/tmp.o7PpafUqPS
+++ rm /tmp/tmp.9hy0eiQFNH /tmp/tmp.o7PpafUqPS
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 1 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RF5abXyuEP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.29p4Id3nTr
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.RF5abXyuEP
+++ cat /tmp/tmp.29p4Id3nTr
+++ rm /tmp/tmp.RF5abXyuEP /tmp/tmp.29p4Id3nTr
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-24467
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1PXcOAQyIx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EQcl9AVEZN
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.1PXcOAQyIx
+++ cat /tmp/tmp.EQcl9AVEZN
+++ rm /tmp/tmp.1PXcOAQyIx /tmp/tmp.EQcl9AVEZN
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vWXYPyuKZ2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dZgtkyCW8Y
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.vWXYPyuKZ2
+++ cat /tmp/tmp.dZgtkyCW8Y
+++ rm /tmp/tmp.vWXYPyuKZ2 /tmp/tmp.dZgtkyCW8Y
+++ return 0
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FBwrTE9Afx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0mdwUCPf0e
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.FBwrTE9Afx
+++ cat /tmp/tmp.0mdwUCPf0e
+++ rm /tmp/tmp.FBwrTE9Afx /tmp/tmp.0mdwUCPf0e
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.JQTG35z69G
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.jXrn8Wf9xX
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.JQTG35z69G
++++ cat /tmp/tmp.jXrn8Wf9xX
++++ rm /tmp/tmp.JQTG35z69G /tmp/tmp.jXrn8Wf9xX
++++ return 0
+++ kubectl_bin -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RCBESov9G4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.98xVFf2ITj
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.RCBESov9G4
+++ cat /tmp/tmp.98xVFf2ITj
+++ rm /tmp/tmp.RCBESov9G4 /tmp/tmp.98xVFf2ITj
+++ return 0
++ echo postgres_password
+ run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.upgrade-24467
+ local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);'
+ local uri=postgres:postgres_password@some-name.upgrade-24467
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.loPfw70DcE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vaukj58L4X
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.loPfw70DcE
++ cat /tmp/tmp.vaukj58L4X
++ rm /tmp/tmp.loPfw70DcE /tmp/tmp.vaukj58L4X
++ return 0
+ local client_container=pg-client-76df9df876-4gvq6
+ kubectl_bin exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ihMm6MNOsz
++ mktemp
+ local LAST_ERR=/tmp/tmp.APOfPYZxqZ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ihMm6MNOsz
+ cat /tmp/tmp.APOfPYZxqZ
+ rm /tmp/tmp.ihMm6MNOsz /tmp/tmp.APOfPYZxqZ
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-24467
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.e6kPGiJLjI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vfFQBPUzle
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.e6kPGiJLjI
+++ cat /tmp/tmp.vfFQBPUzle
+++ rm /tmp/tmp.e6kPGiJLjI /tmp/tmp.vfFQBPUzle
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.nEcOEh8oFb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rkpI8QxkbD
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.nEcOEh8oFb
+++ cat /tmp/tmp.rkpI8QxkbD
+++ rm /tmp/tmp.nEcOEh8oFb /tmp/tmp.rkpI8QxkbD
+++ return 0
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Qjk11wQunP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1X5uhltBmw
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.Qjk11wQunP
+++ cat /tmp/tmp.1X5uhltBmw
+++ rm /tmp/tmp.Qjk11wQunP /tmp/tmp.1X5uhltBmw
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.OdmesqpTjc
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.fGArHdS5DH
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.OdmesqpTjc
++++ cat /tmp/tmp.fGArHdS5DH
++++ rm /tmp/tmp.OdmesqpTjc /tmp/tmp.fGArHdS5DH
++++ return 0
+++ kubectl_bin -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.p7aXpw4ZCW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.y7b9g5rhh7
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.p7aXpw4ZCW
+++ cat /tmp/tmp.y7b9g5rhh7
+++ rm /tmp/tmp.p7aXpw4ZCW /tmp/tmp.y7b9g5rhh7
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.upgrade-24467
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)'
+ local uri=postgres:postgres_password@some-name.upgrade-24467
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Rzhh8l4Vs9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.TzDNgbO2WL
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Rzhh8l4Vs9
++ cat /tmp/tmp.TzDNgbO2WL
++ rm /tmp/tmp.Rzhh8l4Vs9 /tmp/tmp.TzDNgbO2WL
++ return 0
+ local client_container=pg-client-76df9df876-4gvq6
+ kubectl_bin exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.upOtXj1dm9
++ mktemp
+ local LAST_ERR=/tmp/tmp.IDcoBzpxt3
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.upOtXj1dm9
+ cat /tmp/tmp.IDcoBzpxt3
+ rm /tmp/tmp.upOtXj1dm9 /tmp/tmp.IDcoBzpxt3
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-24467
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.cNuUpY6ByQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.F0j302Febm
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.cNuUpY6ByQ
+++ cat /tmp/tmp.F0j302Febm
+++ rm /tmp/tmp.cNuUpY6ByQ /tmp/tmp.F0j302Febm
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.G6Y7X7hzvx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.jtd5QvDKkV
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.G6Y7X7hzvx
+++ cat /tmp/tmp.jtd5QvDKkV
+++ rm /tmp/tmp.G6Y7X7hzvx /tmp/tmp.jtd5QvDKkV
+++ return 0
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.IUoBYU3eb9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wl5twJSC2s
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.IUoBYU3eb9
+++ cat /tmp/tmp.wl5twJSC2s
+++ rm /tmp/tmp.IUoBYU3eb9 /tmp/tmp.wl5twJSC2s
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.mGBS5QMDoy
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.ppBuPsiEzF
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.mGBS5QMDoy
++++ cat /tmp/tmp.ppBuPsiEzF
++++ rm /tmp/tmp.mGBS5QMDoy /tmp/tmp.ppBuPsiEzF
++++ return 0
+++ kubectl_bin -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tNBLnwiXXB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SCb6E57ve2
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.tNBLnwiXXB
+++ cat /tmp/tmp.SCb6E57ve2
+++ rm /tmp/tmp.tNBLnwiXXB /tmp/tmp.SCb6E57ve2
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.upgrade-24467
+ local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";'
+ local uri=postgres:postgres_password@some-name.upgrade-24467
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.xAXWzogwLX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.BtgaDqJUpG
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.xAXWzogwLX
++ cat /tmp/tmp.BtgaDqJUpG
++ rm /tmp/tmp.xAXWzogwLX /tmp/tmp.BtgaDqJUpG
++ return 0
+ local client_container=pg-client-76df9df876-4gvq6
+ kubectl_bin exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ukahxnkP2j
++ mktemp
+ local LAST_ERR=/tmp/tmp.bfwI3u0Jjr
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-24467.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ukahxnkP2j
+ cat /tmp/tmp.bfwI3u0Jjr
+ rm /tmp/tmp.ukahxnkP2j /tmp/tmp.bfwI3u0Jjr
+ return 0
+ sleep 10
+ [[ xinstall == \x\u\p\d\a\t\e ]]
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ejZkpq8sPe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.KqP83SKFnK
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.ejZkpq8sPe
++ cat /tmp/tmp.KqP83SKFnK
++ rm /tmp/tmp.ejZkpq8sPe /tmp/tmp.KqP83SKFnK
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yaCSlwoerF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.r3oKCsneZC
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.yaCSlwoerF
+++ cat /tmp/tmp.r3oKCsneZC
+++ rm /tmp/tmp.yaCSlwoerF /tmp/tmp.r3oKCsneZC
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name
+ sleep 10
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0GORLTrepa
++ mktemp
+ local LAST_ERR=/tmp/tmp.yQs4aQE0D5
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.0GORLTrepa
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.yQs4aQE0D5
+ rm /tmp/tmp.0GORLTrepa /tmp/tmp.yQs4aQE0D5
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oCaCxVzwpm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.gV9zkXpdbC
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.oCaCxVzwpm
++ cat /tmp/tmp.gV9zkXpdbC
++ rm /tmp/tmp.oCaCxVzwpm /tmp/tmp.gV9zkXpdbC
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.E3okC7Hj2t
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RzJ6J2S5ZV
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.E3okC7Hj2t
+++ cat /tmp/tmp.RzJ6J2S5ZV
+++ rm /tmp/tmp.E3okC7Hj2t /tmp/tmp.RzJ6J2S5ZV
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name-pgbouncer..
+ operator_action=update
+ create_or_upgrade_to_version some-name 1.5.1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update
+ local cluster=some-name
+ local pgo_tag=1.5.1
+ local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=update
+ IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.1-postgres-operator
+ IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.1-pgo-deployer
+ IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer
+ IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha
+ IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger
+ [[ xupdate == \x\i\n\s\t\a\l\l ]]
+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local action=update
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update upgrade-24467 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=update
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-24467"#g'
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-24467"#g'
+ cat /dev/fd/63 -
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
++ echo percona/percona-postgresql-operator:1.5.1
++ cut -d: -f1
++ cut -d: -f2
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g'
++ echo 'crunchy_debug: "true"'
++ echo percona/percona-postgresql-operator:1.5.1
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.1"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/1.5.0_operator.yaml metadata.namespace upgrade-24467
+ yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.1-pgo-deployer
+ yq w -d3 - 'subjects[0].namespace' upgrade-24467
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update
++ cat /tmp/tmp.irtas5OIUp/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-24467"
namespace_mode: "dynamic"
pgo_image_prefix: "percona/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "1.5.1"
pgo_operator_namespace: "upgrade-24467"'
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fCJxZLP7RA
++ mktemp
+ local LAST_ERR=/tmp/tmp.DE22zHuknb
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.fCJxZLP7RA
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.DE22zHuknb
+ rm /tmp/tmp.fCJxZLP7RA /tmp/tmp.DE22zHuknb
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 32 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 33 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 34 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZiJH9WR5ey
++ mktemp
+ local LAST_ERR=/tmp/tmp.OfkO7aVU8I
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ZiJH9WR5ey
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.OfkO7aVU8I
+ rm /tmp/tmp.ZiJH9WR5ey /tmp/tmp.OfkO7aVU8I
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nDRx7LwcCC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tbPaG6jjHI
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.nDRx7LwcCC
++ cat /tmp/tmp.tbPaG6jjHI
++ rm /tmp/tmp.nDRx7LwcCC /tmp/tmp.tbPaG6jjHI
++ return 0
+ wait_pod postgres-operator-7696dfc7c7-9qn44
+ local pod=postgres-operator-7696dfc7c7-9qn44
+ local ns=
+ set +o xtrace
postgres-operator-7696dfc7c7-9qn44true
+ [[ -n '' ]]
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6hMqTOcvxW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.j8ZYaRLHks
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.6hMqTOcvxW
++ cat /tmp/tmp.j8ZYaRLHks
++ rm /tmp/tmp.6hMqTOcvxW /tmp/tmp.j8ZYaRLHks
++ return 0
+ [[ xsome-name == \x ]]
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
				{"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"},
				{"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"},
				{"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"},
				{"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"},
				{"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"},
				{"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"}
			]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1pMpym7fXZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.MEPt8j4hOO
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
				{"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"},
				{"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"},
				{"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"},
				{"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"},
				{"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"},
				{"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"}
			]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.1pMpym7fXZ
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.MEPt8j4hOO
+ rm /tmp/tmp.1pMpym7fXZ /tmp/tmp.MEPt8j4hOO
+ return 0
+ [[ xupdate == \x\u\p\d\a\t\e ]]
+ compare_generation 2 deployment some-name-pgbouncer
+ local generation=2
+ local resource=deployment
+ local name=some-name-pgbouncer
+ local current_generation
++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.pz2Ol8kT2s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O6h1Zzu8VR
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.pz2Ol8kT2s
++ cat /tmp/tmp.O6h1Zzu8VR
++ rm /tmp/tmp.pz2Ol8kT2s /tmp/tmp.O6h1Zzu8VR
++ return 0
+ current_generation=1
+ set +o xtrace
.................................................................................................................+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1FaMLWmLO6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.yQg6Z36tOJ
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.1FaMLWmLO6
++ cat /tmp/tmp.yQg6Z36tOJ
++ rm /tmp/tmp.1FaMLWmLO6 /tmp/tmp.yQg6Z36tOJ
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.B8mETkpP9i
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.s8Lz1eVQcU
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.B8mETkpP9i
+++ cat /tmp/tmp.s8Lz1eVQcU
+++ rm /tmp/tmp.B8mETkpP9i /tmp/tmp.s8Lz1eVQcU
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name
+ sleep 10
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.0frNXXcY5V
++ mktemp
+ local LAST_ERR=/tmp/tmp.3bR4vTwuVw
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.0frNXXcY5V
perconapgcluster.pg.percona.com/some-name patched (no change)
+ cat /tmp/tmp.3bR4vTwuVw
+ rm /tmp/tmp.0frNXXcY5V /tmp/tmp.3bR4vTwuVw
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.N7AKwt0Nht
+++ mktemp
++ local LAST_ERR=/tmp/tmp.M4Hq9gDkA4
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.N7AKwt0Nht
++ cat /tmp/tmp.M4Hq9gDkA4
++ rm /tmp/tmp.N7AKwt0Nht /tmp/tmp.M4Hq9gDkA4
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KP3rkvlFXc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.PBHdu2Cxfe
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.KP3rkvlFXc
+++ cat /tmp/tmp.PBHdu2Cxfe
+++ rm /tmp/tmp.KP3rkvlFXc /tmp/tmp.PBHdu2Cxfe
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name-pgbouncer
+ LATEST_TAG=1.6.0
+ IMAGE_OPERATOR=perconalab/percona-postgresql-operator:PR-721-a67b3475b-postgres-operator
+ IMAGE_DEPLOYER=perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer
+ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ IMAGE_PG_HA=perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ desc 'update operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
update operator
-----------------------------------------------------------------------------------

+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled update
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local action=update
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled update upgrade-24467 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local operator_action=update
+ local namespace=upgrade-24467
+ local dont_send_telemetry=false
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-24467"#g'
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ cat /dev/fd/63 -
++ cut -d: -f2
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-24467"#g'
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
++ cut -d: -f1
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-721-a67b3475b"#g'
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g'
++ echo 'crunchy_debug: "true"'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml metadata.namespace upgrade-24467
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update
+ yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer
++ cat /tmp/tmp.irtas5OIUp/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-24467"
namespace_mode: "disabled"
pgo_image_prefix: "perconalab/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "PR-721-a67b3475b"
pgo_operator_namespace: "upgrade-24467"'
+ yq w -d3 - 'subjects[0].namespace' upgrade-24467
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.sUXZyI6XAE
++ mktemp
+ local LAST_ERR=/tmp/tmp.P9rAI5MRt6
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.sUXZyI6XAE
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.P9rAI5MRt6
+ rm /tmp/tmp.sUXZyI6XAE /tmp/tmp.P9rAI5MRt6
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 32 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 33 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.nyssrw7gGX
++ mktemp
+ local LAST_ERR=/tmp/tmp.7bpWaNpfBT
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.nyssrw7gGX
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.7bpWaNpfBT
+ rm /tmp/tmp.nyssrw7gGX /tmp/tmp.7bpWaNpfBT
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.v5iT4YPy4K
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lZj552bhTQ
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.v5iT4YPy4K
++ cat /tmp/tmp.lZj552bhTQ
++ rm /tmp/tmp.v5iT4YPy4K /tmp/tmp.lZj552bhTQ
++ return 0
+ wait_pod postgres-operator-79bc5c49f-676g7
+ local pod=postgres-operator-79bc5c49f-676g7
+ local ns=
+ set +o xtrace
postgres-operator-79bc5c49f-676g7true
+ [[ -n '' ]]
+ kubectl_bin diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.srHni2yMt5
++ mktemp
+ local LAST_ERR=/tmp/tmp.AWoz7rCra0
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.srHni2yMt5
+ cat /tmp/tmp.AWoz7rCra0
+ rm /tmp/tmp.srHni2yMt5 /tmp/tmp.AWoz7rCra0
+ return 0
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo},
			{"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest},
			{"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger},
			{"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer},
			{"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha},
			{"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ugQKylpDQS
++ mktemp
+ local LAST_ERR=/tmp/tmp.WGsgA4y0tv
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo},
			{"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest},
			{"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger},
			{"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer},
			{"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha},
			{"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ugQKylpDQS
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.WGsgA4y0tv
+ rm /tmp/tmp.ugQKylpDQS /tmp/tmp.WGsgA4y0tv
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UX3haXZOpd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vkxCSW8fs6
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.UX3haXZOpd
++ cat /tmp/tmp.vkxCSW8fs6
++ rm /tmp/tmp.UX3haXZOpd /tmp/tmp.vkxCSW8fs6
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3YsyJ9m1Op
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5DBEWQYhHs
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.3YsyJ9m1Op
+++ cat /tmp/tmp.5DBEWQYhHs
+++ rm /tmp/tmp.3YsyJ9m1Op /tmp/tmp.5DBEWQYhHs
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name
+ compare_generation 3 deployment some-name-pgbouncer
+ local generation=3
+ local resource=deployment
+ local name=some-name-pgbouncer
+ local current_generation
++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LPR9W6hskY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GJaszN4AS0
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.LPR9W6hskY
++ cat /tmp/tmp.GJaszN4AS0
++ rm /tmp/tmp.LPR9W6hskY /tmp/tmp.GJaszN4AS0
++ return 0
+ current_generation=2
+ set +o xtrace
.......................................................................................++ kubectl_bin get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TO4FFhDZM4
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ctXIpR3Jh7
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.TO4FFhDZM4
++ cat /tmp/tmp.ctXIpR3Jh7
++ rm /tmp/tmp.TO4FFhDZM4 /tmp/tmp.ctXIpR3Jh7
++ return 0
+ [[ 1.6.0 != 1.6.0 ]]
++ kubectl_bin get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.gwcvmN27qG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ELOQGf9KEY
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.gwcvmN27qG
++ cat /tmp/tmp.ELOQGf9KEY
++ rm /tmp/tmp.gwcvmN27qG /tmp/tmp.ELOQGf9KEY
++ return 0
+ [[ 1.6.0 != 1.6.0 ]]
+ sleep 10
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3GQdyUiRj0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lK7lG9a1Mn
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.3GQdyUiRj0
++ cat /tmp/tmp.lK7lG9a1Mn
++ rm /tmp/tmp.3GQdyUiRj0 /tmp/tmp.lK7lG9a1Mn
++ return 0
+ local client_container=pg-client-76df9df876-4gvq6
+ kubectl_bin exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.GrSC2diuji
++ mktemp
+ local LAST_ERR=/tmp/tmp.THkD8RCwAc
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.GrSC2diuji
+ cat /tmp/tmp.THkD8RCwAc
+ rm /tmp/tmp.GrSC2diuji /tmp/tmp.THkD8RCwAc
+ return 0
+ '[' '!' -s /tmp/tmp.irtas5OIUp/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.irtas5OIUp/select-1.sql
+ backup_time_interval_min=5
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]},
			{"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2kHH8pXrhL
++ mktemp
+ local LAST_ERR=/tmp/tmp.yYey1YYQEx
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]},
			{"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2kHH8pXrhL
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.yYey1YYQEx
+ rm /tmp/tmp.2kHH8pXrhL /tmp/tmp.yYey1YYQEx
+ return 0
+ sleep 10
++ next_backup_after_sec some-name-5min-backup
++ local conf_map_name=some-name-5min-backup
++++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.enIfyqDiSr
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.5Thy4usTfs
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.enIfyqDiSr
++++ cat /tmp/tmp.5Thy4usTfs
++++ rm /tmp/tmp.enIfyqDiSr /tmp/tmp.5Thy4usTfs
++++ return 0
+++ /usr/bin/date +%s --date=2024-04-23T13:50:20Z --utc
++ local configmap_creation_time=1713880220
+++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}'
+++ jq .schedule
+++ grep -Eo '[0-9]+'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.RwzGmyMG7l
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xL989dqLBA
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.RwzGmyMG7l
+++ cat /tmp/tmp.xL989dqLBA
+++ rm /tmp/tmp.RwzGmyMG7l /tmp/tmp.xL989dqLBA
+++ return 0
++ local backup_time_interval_sec=300
+++ calculate_next_backup_start 1713880220 300
+++ local start_time_sec=1713880220
+++ local backup_time_interval_sec=300
+++ local next_interval_hop=1713880520
+++ local next_interval_min_edge=20
+++ local next_backup_at_sec=1713880500
+++ echo 1713880500
+++ get_current_epoch_time
++++ curl -sI https://percona.com/
++++ grep date
++++ /usr/bin/sed 's/date: //g'
+++ local 'curent_date=Tue, 23 Apr 2024 13:50:33 GMT
'
++++ /usr/bin/date --date 'Tue, 23 Apr 2024 13:50:33 GMT
' --utc +%s
+++ echo 1713880233
++ local result=267
++ [[ 267 -gt 0 ]]
++ echo 267
+ sleep 267
+ kubectl_bin wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s
++ mktemp
+ local LAST_OUT=/tmp/tmp.ySydZWvz6l
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Ecx7JuBDy
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ySydZWvz6l
job.batch/some-name-full-sch-backup condition met
+ cat /tmp/tmp.9Ecx7JuBDy
+ rm /tmp/tmp.ySydZWvz6l /tmp/tmp.9Ecx7JuBDy
+ return 0
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Zag0JwTfpo
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NeYLEI7FjR
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl -n upgrade-24467 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Zag0JwTfpo
++ cat /tmp/tmp.NeYLEI7FjR
++ rm /tmp/tmp.Zag0JwTfpo /tmp/tmp.NeYLEI7FjR
++ return 0
+ local client_container=pg-client-76df9df876-4gvq6
+ kubectl_bin exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.tUr7hV2RIH
++ mktemp
+ local LAST_ERR=/tmp/tmp.TyUvtQHAN0
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-76df9df876-4gvq6 -n upgrade-24467 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.tUr7hV2RIH
+ cat /tmp/tmp.TyUvtQHAN0
+ rm /tmp/tmp.tUr7hV2RIH /tmp/tmp.TyUvtQHAN0
+ return 0
+ '[' '!' -s /tmp/tmp.irtas5OIUp/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.irtas5OIUp/select-1.sql
+ kubectl create secret generic some-name-ssl-ca --from-file=ca.crt=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/ca.crt
secret/some-name-ssl-ca created
+ kubectl create secret tls some-name-ssl-keypair --cert=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/server.crt --key=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/server.key
secret/some-name-ssl-keypair created
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7xkmzr8wiE
++ mktemp
+ local LAST_ERR=/tmp/tmp.33hCcaGq2u
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.7xkmzr8wiE
deployment.apps "pg-client" deleted
+ cat /tmp/tmp.33hCcaGq2u
+ rm /tmp/tmp.7xkmzr8wiE /tmp/tmp.33hCcaGq2u
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PUGBr0Bxqs
++ mktemp
+ local LAST_ERR=/tmp/tmp.FBuxZFNY0I
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.PUGBr0Bxqs
deployment.apps/pg-client created
+ cat /tmp/tmp.FBuxZFNY0I
+ rm /tmp/tmp.PUGBr0Bxqs /tmp/tmp.FBuxZFNY0I
+ return 0
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/tlsOnly","value":true},
			{"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''},
			{"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''},
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.lt8M6f6UtN
++ mktemp
+ local LAST_ERR=/tmp/tmp.HIgZsf0fUo
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/tlsOnly","value":true},
			{"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''},
			{"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''},
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.lt8M6f6UtN
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.HIgZsf0fUo
+ rm /tmp/tmp.lt8M6f6UtN /tmp/tmp.HIgZsf0fUo
+ return 0
+ disable_pgBouncer some-name
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.D6DJlGlYMw
++ mktemp
+ local LAST_ERR=/tmp/tmp.hIpiXI7DBl
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.D6DJlGlYMw
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.hIpiXI7DBl
+ rm /tmp/tmp.D6DJlGlYMw /tmp/tmp.hIpiXI7DBl
+ return 0
+ kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.icgPFPDUrJ
++ mktemp
+ local LAST_ERR=/tmp/tmp.jt7FVYxNT9
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.icgPFPDUrJ
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.jt7FVYxNT9
+ rm /tmp/tmp.icgPFPDUrJ /tmp/tmp.jt7FVYxNT9
+ return 0
+ enable_pgBouncer some-name
+ local cluster_name=some-name
+ local replicas_num=1
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.uNJFxWScDQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.szF1SaIx46
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.uNJFxWScDQ
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.szF1SaIx46
+ rm /tmp/tmp.uNJFxWScDQ /tmp/tmp.szF1SaIx46
+ return 0
+ sleep 60
+ kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.FoskgGmvVC
++ mktemp
+ local LAST_ERR=/tmp/tmp.Um1sPJNaRl
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.FoskgGmvVC
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.Um1sPJNaRl
+ rm /tmp/tmp.FoskgGmvVC /tmp/tmp.Um1sPJNaRl
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-24467
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9wy9qREwqv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.GQ1qm2DBMe
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.9wy9qREwqv
++ cat /tmp/tmp.GQ1qm2DBMe
++ rm /tmp/tmp.9wy9qREwqv /tmp/tmp.GQ1qm2DBMe
++ return 0
+ status_query='kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FT5t3pjak0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YGERCGsqi1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+++ kubectl -n upgrade-24467 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.FT5t3pjak0
+++ cat /tmp/tmp.YGERCGsqi1
+++ rm /tmp/tmp.FT5t3pjak0 /tmp/tmp.YGERCGsqi1
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-24467
+ sleep 10
+ set +o xtrace
some-name-pgbouncer
+ sleep 10
+ compare_psql select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467 verify-ca
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local sslmode=verify-ca
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql
+ run_psql_ssl '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-24467 postgres '' verify-ca
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-24467
+ local driver=postgres
++ kubectl_bin get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6JayjyqQxN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8IAuyqOgcP
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.6JayjyqQxN
++ cat /tmp/tmp.8IAuyqOgcP
++ rm /tmp/tmp.6JayjyqQxN /tmp/tmp.8IAuyqOgcP
++ return 0
+ local client_container=pg-client-7f96cb6f54-6w6sq
+ local sslmode=verify-ca
+ kubectl_bin exec pg-client-7f96cb6f54-6w6sq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467'
++ mktemp
+ local LAST_OUT=/tmp/tmp.bOzPrF4wHV
++ mktemp
+ local LAST_ERR=/tmp/tmp.gQmRXrNXiI
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl exec pg-client-7f96cb6f54-6w6sq -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-24467'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.bOzPrF4wHV
+ cat /tmp/tmp.gQmRXrNXiI
+ rm /tmp/tmp.bOzPrF4wHV /tmp/tmp.gQmRXrNXiI
+ return 0
+ '[' '!' -s /tmp/tmp.irtas5OIUp/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.irtas5OIUp/select-1.sql
+ destroy upgrade-24467
++ get_operator_pod
+ grep -v level=debug
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+ grep -v 'Getting tasks for pod'
+ grep -v 'Getting pods from source'
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+++ mktemp
+ grep -v 'the object has been modified'
+ sort -u
+ tee /tmp/tmp.irtas5OIUp/operator.log
++ local LAST_OUT=/tmp/tmp.JWYsNigQCS
+ grep -v level=info
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ExNdi2m9Ti
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.JWYsNigQCS
++ cat /tmp/tmp.ExNdi2m9Ti
++ rm /tmp/tmp.JWYsNigQCS /tmp/tmp.ExNdi2m9Ti
++ return 0
+ kubectl_bin logs postgres-operator-79bc5c49f-676g7 -c operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jbeLQIIgSX
++ mktemp
+ local LAST_ERR=/tmp/tmp.bUhz8Cypxx
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl logs postgres-operator-79bc5c49f-676g7 -c operator
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.jbeLQIIgSX
+ cat /tmp/tmp.bUhz8Cypxx
+ rm /tmp/tmp.jbeLQIIgSX /tmp/tmp.bUhz8Cypxx
+ return 0

  
    
      
          
            
                
                  
                    
                      
                        
                            
                                        
                                            
                                            ]
                                        {
                                        }
                                        },
                                    ]
                                },
                            }
                          }
                        {
                        }
                        },
                      ]
                      {
                      }
                      },
                    ]
                    ],
                    {
                    }
                    },
                    }, {
                    },{
                    }]
                    }],
                  {
                  }
                  },
                ,
                ],
                }
                },
                }],
            }
            },
        }
        },
      {
      }
    ],
    }
    },
  }
  },
 },
{
}
},
2024/04/23 13:50:02 INF    1 (localhost:4150) connecting to nsqd
2024/04/23 13:55:00 INF    2 (localhost:4150) connecting to nsqd
2024/04/23 13:55:28 INF    3 (localhost:4150) connecting to nsqd
2024/04/23 13:55:51 INF    4 (localhost:4150) connecting to nsqd
2024/04/23 13:57:38 INF    5 (localhost:4150) connecting to nsqd
2024/04/23 13:58:17 INF    6 (localhost:4150) connecting to nsqd
2024/04/23 13:58:23 INF    7 (localhost:4150) connecting to nsqd
2024/04/23 13:58:40 INF    8 (localhost:4150) connecting to nsqd
                "affinity": 
                "affinity": {                 "podAntiAffinity": {
                      "allowPrivilegeEscalation": false,
                "annotations": {"keep-backups":"false","keep-data":"false"},
    "apiVersion": "apps/v1",
    "apiVersion": "batch/v1",
  "apiVersion": "v1",
                    "backrest-command": "backup",
                          "configMap": {
                        "containerPort": 5432,
                "containers": [{
    "cpu": "1",
                                                "crunchydata"
                    "crunchy-pgbouncer": "true",
                "crunchy-pgbouncer": "true",
            "crunchy-pgbouncer": "true",
                            "defaultMode": 288
                    "defaultMode": 288,
                "dnsPolicy": "ClusterFirst"
                      "emptyDir": {
                    "emptyDir": {
                    "env": [{
                                "fieldPath": "metadata.namespace"
                            "fieldRef": {
                  "fsGroup": 2,
I0423 13:46:22.530869       1 shared_informer.go:240] Waiting for caches to sync for namespace
I0423 13:46:22.631328       1 shared_informer.go:247] Caches are synced for namespace 
I0423 13:46:22.783774       1 shared_informer.go:240] Waiting for caches to sync for upgrade-24467
I0423 13:46:22.885530       1 shared_informer.go:247] Caches are synced for upgrade-24467 
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer",
                    "imagePullPolicy": "IfNotPresent",
                                            "key": "crunchy-pgbouncer",
                                "key": "password"
                                            "key": "pg-cluster",
                                            "key": "pgo-backrest-job",
                                            "key": "pgo-pg-database",
                                            "key": "pg-pod-anti-affinity",
                                            "key": "vendor",
    "kind": "Deployment",
    "kind": "Job",
  "kind": "Service",
                "labels": {
        "labels": {
      "labels": {
                                "labelSelector": {
                                    "matchExpressions": [
            "matchLabels": {
                "maxSurge": 1
                "maxUnavailable": 1,
                        "medium": "Memory",
                      "medium": "Memory",
    "memory": "128Mi"
            "metadata": {
    "metadata": {
  "metadata": {
                        "mountPath": "/pgconf/",
                        "mountPath": "/pgconf/tls/pgbouncer",
                            "mountPath": "/tmp",
                          "mountPath": "/tmp",
                    "name": "backrest",
                        "name": "COMMAND",
                        "name": "COMMAND_OPTS",
                        "name": "MODE",
                        "name": "NAMESPACE",
                        "name": "PGBACKREST_DB_PATH",
                        "name": "PGBACKREST_LOG_PATH",
                        "name": "PGBACKREST_REPO1_PATH",
                        "name": "PGBACKREST_REPO1_TYPE",
                        "name": "PGBACKREST_STANZA",
                        "name": "pgbouncer"
                    "name": "pgbouncer",
                        "name": "pgbouncer-conf",
                    "name": "pgbouncer-conf",
                        "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
                        "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
                        "name": "PGHA_PGBACKREST_S3_VERIFY_TLS",
                        "name": "PG_PASSWORD",
                        "name": "PG_PRIMARY_SERVICE_NAME",
                        "name": "PITR_TARGET",
                        "name": "PODNAME",
        "name": "postgres",
                "name": "some-name-full-sch-backup",
                    "name": "some-name-pgbouncer",
                "name": "some-name-pgbouncer",
            "name": "some-name-pgbouncer",
          "name": "some-name-pgbouncer"
        "name": "some-name-pgbouncer",
      "name": "some-name-pgbouncer",
                            "name": "some-name-pgbouncer-cm"
                                "name": "some-name-pgbouncer-secret",
                            "name": "some-name-pgbouncer-secret",
                              "name": "some-name-ssl-ca"
                              "name": "some-name-ssl-keypair"
                        "name": "tls-pgbouncer"
                    "name": "tls-pgbouncer",
                            "name": "tmp"
                          "name": "tmp"
                      "name": "tmp",
                    "name": "tmp",
        "nodePort": 0
                                            "operator": "Exists"
                                            "operator": "In",
                    "pg-cluster": "some-name"
                    "pg-cluster": "some-name",
                "pg-cluster": "some-name",
            "pg-cluster": "some-name",
          "pg-cluster": "some-name",
                    "pgo-backrest-job": "true",
                    "pgo-backrest": "true",
                    "pg-pod-anti-affinity": "preferred",
                "pg-pod-anti-affinity": "preferred",
                            "podAffinityTerm": {
                                        "podAntiAffinity": {
                "podAntiAffinity": {
        "port": 5432,
                    "ports": [{
    "ports": [
                    "preferredDuringSchedulingIgnoredDuringExecution": [
                      "privileged": false,
                    "projected": {
                        "protocol": "TCP",
        "protocol": "TCP",
                        "readOnly": false
                      "readOnlyRootFilesystem": true
        "replicas": 1,
  "requests": {
"resources": {
                "restartPolicy": "Always",
                "restartPolicy": "Never"
            "rollingUpdate": {
                  "runAsNonRoot": true
                          "secret": {
                            "secretKeyRef": {
                    "securityContext": {
                "securityContext": {
                "securityContext": {},
        "selector": {
    "selector": {
                "serviceAccountName": "pgo-backrest",
                "serviceAccountName": "pgo-default",
                    "service-name": "some-name-pgbouncer",
                "service-name": "some-name-pgbouncer",
            "service-name": "some-name-pgbouncer",
      "service-name": "some-name-pgbouncer"
    "sessionAffinity": "None"
                        "sizeLimit": "16Mi"
                      "sizeLimit": "1Mi"
                                                "some-name"
                      "sources": [
            "spec": {
    "spec": {
  "spec": {
        "strategy": {
        "targetPort": 5432,
        "template": {
time="2024-04-23T13:46:22Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0
time="2024-04-23T13:46:23Z" level=error msg="create pgcluster resource: create pgcluster resource: pgclusters.pg.percona.com \"some-name\" already exists" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgc.(*Controller).onAdd()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgc/pgc.go:174" version=1.6.0
time="2024-04-23T13:49:45Z" level=error msg="command terminated with exit code 2" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T13:49:47Z" level=error msg="update perconapgcluster: check is pgcluster primary: no role labels in pod some-name-78645dc78c-gj5xz" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgc.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgc/pgc.go:532" version=1.6.0
time="2024-04-23T13:50:01Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T13:50:01Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name-repl1/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
time="2024-04-23T13:56:22Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0
time="2024-04-23T13:56:22Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T13:56:22Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0
time="2024-04-23T13:57:37Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T13:57:37Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
                                "topologyKey": "kubernetes.io/hostname"
    "type": "ClusterIP",
            "type": "RollingUpdate",
                        "value": ""
                        "value": "backup"
                        "value": "false"
                        "valueFrom": {
                        "value": "pgbackrest"
                        "value": "posix"
                                            "values": [
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-xbsgz"
                        "value": "--stanza=db --type=full  --repo1-retention-full=3 --db-host=10.251.72.12 --db-path=/pgdata/some-name"
                        "value": "/tmp"
                    "vendor": "crunchydata"
                    "vendor": "crunchydata",
                "vendor": "crunchydata"
            "vendor": "crunchydata"
          "vendor": "crunchydata",
                    "volumeMounts": [
                "volumes": [
                            "weight": 1,
+ destroy_operator
+ kubectl_bin delete -f /tmp/tmp.irtas5OIUp/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.F1REtWjYf2
++ mktemp
+ local LAST_ERR=/tmp/tmp.M9dRB0XUda
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.F1REtWjYf2
+ cat /tmp/tmp.M9dRB0XUda
Error from server (NotFound): error when deleting "/tmp/tmp.irtas5OIUp/operator.yaml": serviceaccounts "pgo-deployer-sa" not found
Error from server (NotFound): error when deleting "/tmp/tmp.irtas5OIUp/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): error when deleting "/tmp/tmp.irtas5OIUp/operator.yaml": configmaps "pgo-deployer-cm" not found
Error from server (NotFound): error when deleting "/tmp/tmp.irtas5OIUp/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found
Error from server (NotFound): error when deleting "/tmp/tmp.irtas5OIUp/operator.yaml": jobs.batch "pgo-deploy" not found
+ rm /tmp/tmp.F1REtWjYf2 /tmp/tmp.M9dRB0XUda
+ return 1
+ true
+ yq w -d4 /tmp/tmp.irtas5OIUp/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall
+ kubectl_bin apply -f /tmp/tmp.irtas5OIUp/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2l80rvYPjg
++ mktemp
+ local LAST_ERR=/tmp/tmp.yqUSrgx7VY
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl apply -f /tmp/tmp.irtas5OIUp/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2l80rvYPjg
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.yqUSrgx7VY
+ rm /tmp/tmp.2l80rvYPjg /tmp/tmp.yqUSrgx7VY
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-24467
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-24467 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.irtas5OIUp/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.fM4UrGYalp
++ mktemp
+ local LAST_ERR=/tmp/tmp.z71MvxFOBJ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete -f /tmp/tmp.irtas5OIUp/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.fM4UrGYalp
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.z71MvxFOBJ
+ rm /tmp/tmp.fM4UrGYalp /tmp/tmp.z71MvxFOBJ
+ return 0
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns --grace-period=0 --force=true
+ local LAST_OUT=/tmp/tmp.qkd25QIYeP
++ mktemp
+ local LAST_ERR=/tmp/tmp.zb5Jn9DA33
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl get ns
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.qkd25QIYeP
+ cat /tmp/tmp.zb5Jn9DA33
+ rm /tmp/tmp.qkd25QIYeP /tmp/tmp.zb5Jn9DA33
+ return 0
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
namespace "upgrade-24467" force deleted
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.eqDqmAQkPq
++ mktemp
+ local LAST_ERR=/tmp/tmp.HW1sAjf4jY
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.eqDqmAQkPq
+ cat /tmp/tmp.HW1sAjf4jY
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.eqDqmAQkPq /tmp/tmp.HW1sAjf4jY
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.85dJbIIwhx
++ mktemp
+ local LAST_ERR=/tmp/tmp.EBnGQNG2ip
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.85dJbIIwhx
+ cat /tmp/tmp.EBnGQNG2ip
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.85dJbIIwhx /tmp/tmp.EBnGQNG2ip
+ return 1
+ true
+ rm -rf /tmp/tmp.irtas5OIUp