++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/logs/upgrade.log'
Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/logs/upgrade.log
++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP=
++ oc get projects
error: the server doesn't have a resource type "projects"
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+++ helm version -c
+++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/'
++ HELM_VERSION=v3.14.4
++ '[' v3 == v2 ']'
+ main
+ create_namespace upgrade-4817
+ local namespace=upgrade-4817
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ cleanup_rbac
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
++ mktemp
+ xargs kubectl delete ns
+ kubectl_bin get ns
+ local LAST_OUT=/tmp/tmp.jbtfULVXzy
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.ud9T5xdqvm
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ local LAST_OUT=/tmp/tmp.CGbmmHaBNs
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ awk '{print$1}'
++ mktemp
+ local LAST_ERR=/tmp/tmp.TxmZ6MdkQJ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl get ns
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.CGbmmHaBNs
+ cat /tmp/tmp.TxmZ6MdkQJ
E0424 16:25:05.618454   16887 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:05.829163   16887 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:05.935946   16887 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:06.043292   16887 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
+ rm /tmp/tmp.CGbmmHaBNs /tmp/tmp.TxmZ6MdkQJ
+ return 0
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
E0424 16:25:07.869253   17143 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:08.111342   17143 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:08.217294   17143 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:08.323679   17143 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.jbtfULVXzy
+ cat /tmp/tmp.ud9T5xdqvm
E0424 16:25:13.775992   17566 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:13.995883   17566 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:14.101962   17566 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:14.207610   17566 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.jbtfULVXzy /tmp/tmp.ud9T5xdqvm
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
++ mktemp
+ local LAST_OUT=/tmp/tmp.f7oTo7Zlui
++ mktemp
+ local LAST_ERR=/tmp/tmp.OPvyWvrLKx
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.f7oTo7Zlui
+ cat /tmp/tmp.OPvyWvrLKx
E0424 16:25:31.666627   18275 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:31.889547   18275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:31.999606   18275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
E0424 16:25:32.105969   18275 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.f7oTo7Zlui /tmp/tmp.OPvyWvrLKx
+ return 1
+ true
+ kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth
Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found
+ true
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin delete namespace upgrade-4817
++ mktemp
+ local LAST_OUT=/tmp/tmp.OI2y0tv3WY
++ mktemp
+ local LAST_ERR=/tmp/tmp.5TdE3R4rMQ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete namespace upgrade-4817
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete namespace upgrade-4817
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete namespace upgrade-4817
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.OI2y0tv3WY
+ cat /tmp/tmp.5TdE3R4rMQ
Error from server (NotFound): namespaces "upgrade-4817" not found
+ rm /tmp/tmp.OI2y0tv3WY /tmp/tmp.5TdE3R4rMQ
+ return 1
+ :
+ wait_for_delete namespace/upgrade-4817
+ local res=namespace/upgrade-4817
+ set +o xtrace
namespace/upgrade-4817 - Error from server (NotFound): namespaces "upgrade-4817" not found
+ kubectl_bin create namespace upgrade-4817
++ mktemp
+ local LAST_OUT=/tmp/tmp.2vXJdbHWAv
++ mktemp
+ local LAST_ERR=/tmp/tmp.LR4I7wiNcn
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl create namespace upgrade-4817
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2vXJdbHWAv
namespace/upgrade-4817 created
+ cat /tmp/tmp.LR4I7wiNcn
+ rm /tmp/tmp.2vXJdbHWAv /tmp/tmp.LR4I7wiNcn
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sXWtol2qmV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RX7ny93Lwi
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl config current-context
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.sXWtol2qmV
++ cat /tmp/tmp.RX7ny93Lwi
++ rm /tmp/tmp.sXWtol2qmV /tmp/tmp.RX7ny93Lwi
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-upgrade --namespace=upgrade-4817
++ mktemp
+ local LAST_OUT=/tmp/tmp.D0QycEaz8K
++ mktemp
+ local LAST_ERR=/tmp/tmp.c6A0kYWPQX
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-upgrade --namespace=upgrade-4817
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.D0QycEaz8K
Context "gke_cloud-dev-112233_us-central1-c_jen-pg-723-7ed19e708-2-upgrade" modified.
+ cat /tmp/tmp.c6A0kYWPQX
+ rm /tmp/tmp.D0QycEaz8K /tmp/tmp.c6A0kYWPQX
+ return 0
+ cluster=some-name
+ operator_action=install
+ create_or_upgrade_to_version some-name 1.5.0 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install
+ local cluster=some-name
+ local pgo_tag=1.5.0
+ local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=install
+ IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.0-postgres-operator
+ IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.0-pgo-deployer
+ IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer
+ IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha
+ IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger
+ [[ xinstall == \x\i\n\s\t\a\l\l ]]
+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local action=install
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install upgrade-4817 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=install
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-4817"#g'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g'
+ cat /dev/fd/63 -
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-4817"#g'
++ echo 'crunchy_debug: "true"'
++ echo percona/percona-postgresql-operator:1.5.0
++ cut -d: -f1
++ echo percona/percona-postgresql-operator:1.5.0
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g'
++ cut -d: -f2
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.0"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.4.0_operator.yaml metadata.namespace upgrade-4817
+ yq w -d3 - 'subjects[0].namespace' upgrade-4817
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install
+ yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.0-pgo-deployer
++ cat /tmp/tmp.wKxCFsu2FT/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-4817"
namespace_mode: "dynamic"
pgo_image_prefix: "percona/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "1.5.0"
pgo_operator_namespace: "upgrade-4817"'
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.tyd66dJxTS
++ mktemp
+ local LAST_ERR=/tmp/tmp.MOaRQF4G0k
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.tyd66dJxTS
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.MOaRQF4G0k
+ rm /tmp/tmp.tyd66dJxTS /tmp/tmp.MOaRQF4G0k
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.CyrKOmjdef
++ mktemp
+ local LAST_ERR=/tmp/tmp.CBCX62HzwJ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.CyrKOmjdef
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.CBCX62HzwJ
+ rm /tmp/tmp.CyrKOmjdef /tmp/tmp.CBCX62HzwJ
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ovXnKYj1wd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.7i1UTp0XjA
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.ovXnKYj1wd
++ cat /tmp/tmp.7i1UTp0XjA
++ rm /tmp/tmp.ovXnKYj1wd /tmp/tmp.7i1UTp0XjA
++ return 0
+ wait_pod postgres-operator-779646fb6-kzzs9
+ local pod=postgres-operator-779646fb6-kzzs9
+ local ns=
+ set +o xtrace
postgres-operator-779646fb6-kzzs9true
+ [[ -n '' ]]
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fxAMlA7FJb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.osCa127N6u
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 0
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 4
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=1
++ [[ 1 != 0 ]]
++ sleep 8
++ set -o errexit
++ cat /tmp/tmp.fxAMlA7FJb
++ cat /tmp/tmp.osCa127N6u
Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found
++ rm /tmp/tmp.fxAMlA7FJb /tmp/tmp.osCa127N6u
++ return 1
+ [[ x == \x ]]
+ desc 'start cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
start cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name
+ local cluster=some-name
+ local namespace=upgrade-4817
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml
+ yq w - metadata.name some-name-users-secret
+ kubectl_bin -n upgrade-4817 apply -f -
+ yq w - stringData.some-name some-name_pass
++ mktemp
+ local LAST_OUT=/tmp/tmp.v0qlgBFf5M
++ mktemp
+ local LAST_ERR=/tmp/tmp.V7ECrUehHL
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.v0qlgBFf5M
secret/some-name-users-secret created
+ cat /tmp/tmp.V7ECrUehHL
+ rm /tmp/tmp.v0qlgBFf5M /tmp/tmp.V7ECrUehHL
+ return 0
+ yq w - metadata.name some-name-pmm-secret
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml
+ kubectl_bin -n upgrade-4817 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kx2W4RQma3
++ mktemp
+ local LAST_ERR=/tmp/tmp.1NCDVMFwBA
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.kx2W4RQma3
secret/some-name-pmm-secret created
+ cat /tmp/tmp.1NCDVMFwBA
+ rm /tmp/tmp.kx2W4RQma3 /tmp/tmp.1NCDVMFwBA
+ return 0
+ spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local backup=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml
+ local namespace=upgrade-4817
+ local root_via_pgbouncer=
+ desc 'create fresh PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create fresh PG cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name upgrade-4817
+ local cluster=some-name
+ local namespace=upgrade-4817
+ yq w - metadata.name some-name-users-secret
+ kubectl_bin -n upgrade-4817 apply -f -
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml
++ mktemp
+ yq w - stringData.some-name some-name_pass
+ local LAST_OUT=/tmp/tmp.Y7doruZ7Lc
++ mktemp
+ local LAST_ERR=/tmp/tmp.XDca4SHS0E
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Y7doruZ7Lc
secret/some-name-users-secret configured
+ cat /tmp/tmp.XDca4SHS0E
+ rm /tmp/tmp.Y7doruZ7Lc /tmp/tmp.XDca4SHS0E
+ return 0
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/users-secret.yaml
+ yq w - metadata.name some-name-pmm-secret
+ kubectl_bin -n upgrade-4817 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.YODNVXT3t8
++ mktemp
+ local LAST_ERR=/tmp/tmp.UYIxwnKZ9Y
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.YODNVXT3t8
secret/some-name-pmm-secret configured
+ cat /tmp/tmp.UYIxwnKZ9Y
+ rm /tmp/tmp.YODNVXT3t8 /tmp/tmp.UYIxwnKZ9Y
+ return 0
+ kubectl_bin -n upgrade-4817 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Roec2w98Rq
++ mktemp
+ local LAST_ERR=/tmp/tmp.1ebAuYYC4t
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Roec2w98Rq
deployment.apps/pg-client created
+ cat /tmp/tmp.1ebAuYYC4t
+ rm /tmp/tmp.Roec2w98Rq /tmp/tmp.1ebAuYYC4t
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_cr.yaml some-name false false false false false false false false
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.labels.deployment-name some-name
+ yq w - metadata.labels.name some-name
+ yq w - spec.database some-name
+ yq w - spec.name some-name
+ yq w - spec.keepData false
+ yq w - spec.clustername some-name
+ yq w - spec.keepBackups false
+ yq w - spec.user some-name
+ yq w - metadata.name some-name
+ yq w - spec.secretsName some-name-users-secret
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgPrimary.image percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha
+ yq w - spec.backup.backrestRepoImage percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo
+ yq w - spec.pgBouncer.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer
+ yq w - spec.pgBadger.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger
+ yq w - spec.backup.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.wKxCFsu2FT/cr.yaml -n upgrade-4817
++ mktemp
+ local LAST_OUT=/tmp/tmp.X5Tbi1vdS7
++ mktemp
+ local LAST_ERR=/tmp/tmp.qOgpeTP1VP
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /tmp/tmp.wKxCFsu2FT/cr.yaml -n upgrade-4817
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.X5Tbi1vdS7
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.qOgpeTP1VP
+ rm /tmp/tmp.X5Tbi1vdS7 /tmp/tmp.qOgpeTP1VP
+ return 0
+ wait_deployment some-name-backrest-shared-repo
+ local name=some-name-backrest-shared-repo
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name-backrest-shared-repo...
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name...........
+ [[ false == \g\c\s\+* ]]
+ wait_job_completion some-name-stanza-create false
+ local job=some-name-stanza-create
+ local mandatory=false
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
some-name-stanza-create+ retry=0
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name
+ local job=backrest-backup-some-name
+ local mandatory=true
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
backrest-backup-some-name+ retry=0
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MlfD1XpzIH
+++ mktemp
++ local LAST_ERR=/tmp/tmp.tVBPGShsQV
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.MlfD1XpzIH
++ cat /tmp/tmp.tVBPGShsQV
++ rm /tmp/tmp.MlfD1XpzIH /tmp/tmp.tVBPGShsQV
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UpZIH2NoJD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.HHoknmYcRD
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.UpZIH2NoJD
+++ cat /tmp/tmp.HHoknmYcRD
+++ rm /tmp/tmp.UpZIH2NoJD /tmp/tmp.HHoknmYcRD
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 1 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.xR9si3Hscb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Rg86X5dJm0
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.xR9si3Hscb
+++ cat /tmp/tmp.Rg86X5dJm0
+++ rm /tmp/tmp.xR9si3Hscb /tmp/tmp.Rg86X5dJm0
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-4817
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zOGVuSlXoa
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.qsIvVCOkjT
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.zOGVuSlXoa
+++ cat /tmp/tmp.qsIvVCOkjT
+++ rm /tmp/tmp.zOGVuSlXoa /tmp/tmp.qsIvVCOkjT
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.I7iUPJtFLp
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ravYCl1Gbs
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.I7iUPJtFLp
+++ cat /tmp/tmp.ravYCl1Gbs
+++ rm /tmp/tmp.I7iUPJtFLp /tmp/tmp.ravYCl1Gbs
+++ return 0
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AVMUbgIEro
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.283Y9FOEi7
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.AVMUbgIEro
+++ cat /tmp/tmp.283Y9FOEi7
+++ rm /tmp/tmp.AVMUbgIEro /tmp/tmp.283Y9FOEi7
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.8W3UQ0M5My
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.w6IULQzrwT
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.8W3UQ0M5My
++++ cat /tmp/tmp.w6IULQzrwT
++++ rm /tmp/tmp.8W3UQ0M5My /tmp/tmp.w6IULQzrwT
++++ return 0
+++ kubectl_bin -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5ifuzBgvHN
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.t9meY0tvZH
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.5ifuzBgvHN
+++ cat /tmp/tmp.t9meY0tvZH
+++ rm /tmp/tmp.5ifuzBgvHN /tmp/tmp.t9meY0tvZH
+++ return 0
++ echo postgres_password
+ run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.upgrade-4817
+ local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);'
+ local uri=postgres:postgres_password@some-name.upgrade-4817
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vOyWeipJD2
+++ mktemp
++ local LAST_ERR=/tmp/tmp.AlQQI8nm8q
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.vOyWeipJD2
++ cat /tmp/tmp.AlQQI8nm8q
++ rm /tmp/tmp.vOyWeipJD2 /tmp/tmp.AlQQI8nm8q
++ return 0
+ local client_container=pg-client-76df9df876-hkdkb
+ kubectl_bin exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2zOvuABL7E
++ mktemp
+ local LAST_ERR=/tmp/tmp.M7e51DEgCp
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2zOvuABL7E
+ cat /tmp/tmp.M7e51DEgCp
+ rm /tmp/tmp.2zOvuABL7E /tmp/tmp.M7e51DEgCp
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-4817
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.L5c0yLfZuP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.P1P5OLOe4t
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.L5c0yLfZuP
+++ cat /tmp/tmp.P1P5OLOe4t
+++ rm /tmp/tmp.L5c0yLfZuP /tmp/tmp.P1P5OLOe4t
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.mVV7WkoahD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DxxbSj0KnK
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.mVV7WkoahD
+++ cat /tmp/tmp.DxxbSj0KnK
+++ rm /tmp/tmp.mVV7WkoahD /tmp/tmp.DxxbSj0KnK
+++ return 0
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vDsVryzWWL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ARKCbrHRBb
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.vDsVryzWWL
+++ cat /tmp/tmp.ARKCbrHRBb
+++ rm /tmp/tmp.vDsVryzWWL /tmp/tmp.ARKCbrHRBb
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.t2cNtJ98nm
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.GjIY9iip1s
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.t2cNtJ98nm
++++ cat /tmp/tmp.GjIY9iip1s
++++ rm /tmp/tmp.t2cNtJ98nm /tmp/tmp.GjIY9iip1s
++++ return 0
+++ kubectl_bin -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.04kQ2ygKUQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.otLnd97eTe
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.04kQ2ygKUQ
+++ cat /tmp/tmp.otLnd97eTe
+++ rm /tmp/tmp.04kQ2ygKUQ /tmp/tmp.otLnd97eTe
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.upgrade-4817
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)'
+ local uri=postgres:postgres_password@some-name.upgrade-4817
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uTRPOAWqrF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0m9nLroi8T
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.uTRPOAWqrF
++ cat /tmp/tmp.0m9nLroi8T
++ rm /tmp/tmp.uTRPOAWqrF /tmp/tmp.0m9nLroi8T
++ return 0
+ local client_container=pg-client-76df9df876-hkdkb
+ kubectl_bin exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1ZYK8vP3SU
++ mktemp
+ local LAST_ERR=/tmp/tmp.gRlJkuqXgo
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.1ZYK8vP3SU
+ cat /tmp/tmp.gRlJkuqXgo
+ rm /tmp/tmp.1ZYK8vP3SU /tmp/tmp.gRlJkuqXgo
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=upgrade-4817
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.07FPhhK7f4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XcDJ0r2HGF
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.07FPhhK7f4
+++ cat /tmp/tmp.XcDJ0r2HGF
+++ rm /tmp/tmp.07FPhhK7f4 /tmp/tmp.XcDJ0r2HGF
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fSQdXFgLpP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1NvmEWv9rw
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.fSQdXFgLpP
+++ cat /tmp/tmp.1NvmEWv9rw
+++ rm /tmp/tmp.fSQdXFgLpP /tmp/tmp.1NvmEWv9rw
+++ return 0
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kBbW6prv8z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KjN3XI2zvy
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.kBbW6prv8z
+++ cat /tmp/tmp.KjN3XI2zvy
+++ rm /tmp/tmp.kBbW6prv8z /tmp/tmp.KjN3XI2zvy
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.omZ9fd8jgT
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.kHWkj2R83R
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.omZ9fd8jgT
++++ cat /tmp/tmp.kHWkj2R83R
++++ rm /tmp/tmp.omZ9fd8jgT /tmp/tmp.kHWkj2R83R
++++ return 0
+++ kubectl_bin -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qiFuEXmWDi
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1o7k8ZYJHy
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.qiFuEXmWDi
+++ cat /tmp/tmp.1o7k8ZYJHy
+++ rm /tmp/tmp.qiFuEXmWDi /tmp/tmp.1o7k8ZYJHy
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.upgrade-4817
+ local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";'
+ local uri=postgres:postgres_password@some-name.upgrade-4817
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.YaBGxFeD2S
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UfFeTH3bt5
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.YaBGxFeD2S
++ cat /tmp/tmp.UfFeTH3bt5
++ rm /tmp/tmp.YaBGxFeD2S /tmp/tmp.UfFeTH3bt5
++ return 0
+ local client_container=pg-client-76df9df876-hkdkb
+ kubectl_bin exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Rq9XXbHl8H
++ mktemp
+ local LAST_ERR=/tmp/tmp.Le99yVUNoZ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-4817.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Rq9XXbHl8H
+ cat /tmp/tmp.Le99yVUNoZ
+ rm /tmp/tmp.Rq9XXbHl8H /tmp/tmp.Le99yVUNoZ
+ return 0
+ sleep 10
+ [[ xinstall == \x\u\p\d\a\t\e ]]
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2WAJ1WJPbf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OpN7Xft532
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.2WAJ1WJPbf
++ cat /tmp/tmp.OpN7Xft532
++ rm /tmp/tmp.2WAJ1WJPbf /tmp/tmp.OpN7Xft532
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dAGdZMDnSv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EYBfqQx0ge
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.dAGdZMDnSv
+++ cat /tmp/tmp.EYBfqQx0ge
+++ rm /tmp/tmp.dAGdZMDnSv /tmp/tmp.EYBfqQx0ge
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name
+ sleep 10
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.PklM87RYw2
++ mktemp
+ local LAST_ERR=/tmp/tmp.s0SN3iFTXq
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.PklM87RYw2
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.s0SN3iFTXq
+ rm /tmp/tmp.PklM87RYw2 /tmp/tmp.s0SN3iFTXq
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.8vmh7a9Df9
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hSgnS6SLjg
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.8vmh7a9Df9
++ cat /tmp/tmp.hSgnS6SLjg
++ rm /tmp/tmp.8vmh7a9Df9 /tmp/tmp.hSgnS6SLjg
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sxFJZRrJbW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.sxiLLzhQwq
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.sxFJZRrJbW
+++ cat /tmp/tmp.sxiLLzhQwq
+++ rm /tmp/tmp.sxFJZRrJbW /tmp/tmp.sxiLLzhQwq
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name-pgbouncer..
+ operator_action=update
+ create_or_upgrade_to_version some-name 1.5.1 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update
+ local cluster=some-name
+ local pgo_tag=1.5.1
+ local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=update
+ IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.1-postgres-operator
+ IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.1-pgo-deployer
+ IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer
+ IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha
+ IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger
+ [[ xupdate == \x\i\n\s\t\a\l\l ]]
+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local action=update
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update upgrade-4817 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml
+ local namespace_mode=dynamic
+ local operator_action=update
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-4817"#g'
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-4817"#g'
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ cat /dev/fd/63 -
++ cut -d: -f2
++ echo percona/percona-postgresql-operator:1.5.1
++ cut -d: -f1
++ echo percona/percona-postgresql-operator:1.5.1
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g'
++ echo 'crunchy_debug: "true"'
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.1"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update
+ yq w -d3 - 'subjects[0].namespace' upgrade-4817
+ yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.1-pgo-deployer
++ cat /tmp/tmp.wKxCFsu2FT/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-4817"
namespace_mode: "dynamic"
pgo_image_prefix: "percona/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "1.5.1"
pgo_operator_namespace: "upgrade-4817"'
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/1.5.0_operator.yaml metadata.namespace upgrade-4817
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ONfQQAnfqH
++ mktemp
+ local LAST_ERR=/tmp/tmp.5UxdWPvTN5
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ONfQQAnfqH
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.5UxdWPvTN5
+ rm /tmp/tmp.ONfQQAnfqH /tmp/tmp.5UxdWPvTN5
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 32 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 33 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.dFSobh7Rk6
++ mktemp
+ local LAST_ERR=/tmp/tmp.yHKNiu4iBD
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.dFSobh7Rk6
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.yHKNiu4iBD
+ rm /tmp/tmp.dFSobh7Rk6 /tmp/tmp.yHKNiu4iBD
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DNNe2r180o
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rNk96v4epj
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.DNNe2r180o
++ cat /tmp/tmp.rNk96v4epj
++ rm /tmp/tmp.DNNe2r180o /tmp/tmp.rNk96v4epj
++ return 0
+ wait_pod postgres-operator-6bcc96cd6d-2lbkj
+ local pod=postgres-operator-6bcc96cd6d-2lbkj
+ local ns=
+ set +o xtrace
postgres-operator-6bcc96cd6d-2lbkjtrue
+ [[ -n '' ]]
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.esmQwlYR5f
+++ mktemp
++ local LAST_ERR=/tmp/tmp.sih2KbbALB
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.esmQwlYR5f
++ cat /tmp/tmp.sih2KbbALB
++ rm /tmp/tmp.esmQwlYR5f /tmp/tmp.sih2KbbALB
++ return 0
+ [[ xsome-name == \x ]]
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
				{"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"},
				{"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"},
				{"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"},
				{"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"},
				{"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"},
				{"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"}
			]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.N98L5VptHB
++ mktemp
+ local LAST_ERR=/tmp/tmp.SN4QAsfaSR
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
				{"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"},
				{"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"},
				{"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"},
				{"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"},
				{"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"},
				{"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"}
			]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.N98L5VptHB
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.SN4QAsfaSR
+ rm /tmp/tmp.N98L5VptHB /tmp/tmp.SN4QAsfaSR
+ return 0
+ [[ xupdate == \x\u\p\d\a\t\e ]]
+ compare_generation 2 deployment some-name-pgbouncer
+ local generation=2
+ local resource=deployment
+ local name=some-name-pgbouncer
+ local current_generation
++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z3yrsliUKL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8EonQ4TZmh
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.z3yrsliUKL
++ cat /tmp/tmp.8EonQ4TZmh
++ rm /tmp/tmp.z3yrsliUKL /tmp/tmp.8EonQ4TZmh
++ return 0
+ current_generation=1
+ set +o xtrace
.......................................................................................................................+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p94gCuMcGB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.V3XdFtuKJa
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.p94gCuMcGB
++ cat /tmp/tmp.V3XdFtuKJa
++ rm /tmp/tmp.p94gCuMcGB /tmp/tmp.V3XdFtuKJa
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DcEdl1rsPK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vLo7AyaWYs
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.DcEdl1rsPK
+++ cat /tmp/tmp.vLo7AyaWYs
+++ rm /tmp/tmp.DcEdl1rsPK /tmp/tmp.vLo7AyaWYs
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name
+ sleep 10
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.aOukFxsEYk
++ mktemp
+ local LAST_ERR=/tmp/tmp.q9xdG2XAKc
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/pgBouncer/size","value":1}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.aOukFxsEYk
perconapgcluster.pg.percona.com/some-name patched (no change)
+ cat /tmp/tmp.q9xdG2XAKc
+ rm /tmp/tmp.aOukFxsEYk /tmp/tmp.q9xdG2XAKc
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DzmnRAUc4m
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QgtskT9pPR
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.DzmnRAUc4m
++ cat /tmp/tmp.QgtskT9pPR
++ rm /tmp/tmp.DzmnRAUc4m /tmp/tmp.QgtskT9pPR
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qRkX67D07M
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.J8rD5lZbFO
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.qRkX67D07M
+++ cat /tmp/tmp.J8rD5lZbFO
+++ rm /tmp/tmp.qRkX67D07M /tmp/tmp.J8rD5lZbFO
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name-pgbouncer
+ LATEST_TAG=1.6.0
+ IMAGE_OPERATOR=perconalab/percona-postgresql-operator:PR-723-7ed19e708-postgres-operator
+ IMAGE_DEPLOYER=perconalab/percona-postgresql-operator:PR-723-7ed19e708-pgo-deployer
+ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ IMAGE_PG_HA=perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ IMAGE_BACKREST_REPO=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ IMAGE_PGBADGER=perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ desc 'update operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
update operator
-----------------------------------------------------------------------------------

+ deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml disabled update
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml
+ local namespace_mode=disabled
+ local action=update
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml disabled update upgrade-4817 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml
+ local namespace_mode=disabled
+ local operator_action=update
+ local namespace=upgrade-4817
+ local dont_send_telemetry=false
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-4817"#g'
+ cat /dev/fd/63 -
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g'
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-4817"#g'
++ echo 'crunchy_debug: "true"'
++ cut -d: -f1
++ echo perconalab/percona-postgresql-operator:PR-723-7ed19e708
++ cut -d: -f2
++ echo perconalab/percona-postgresql-operator:PR-723-7ed19e708
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g'
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-723-7ed19e708"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w -d3 - 'subjects[0].namespace' upgrade-4817
+ yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-723-7ed19e708-pgo-deployer
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update
++ cat /tmp/tmp.wKxCFsu2FT/operator.ini
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-723/deploy/operator.yaml metadata.namespace upgrade-4817
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "upgrade-4817"
namespace_mode: "disabled"
pgo_image_prefix: "perconalab/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "PR-723-7ed19e708"
pgo_operator_namespace: "upgrade-4817"'
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ahgHNQ3UKF
++ mktemp
+ local LAST_ERR=/tmp/tmp.nsTsaQ0RVS
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ahgHNQ3UKF
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.nsTsaQ0RVS
+ rm /tmp/tmp.ahgHNQ3UKF /tmp/tmp.nsTsaQ0RVS
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ grep -io 'not found'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uhMuNTNDpk
++ mktemp
+ local LAST_ERR=/tmp/tmp.jwXH5bNYBc
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.uhMuNTNDpk
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.jwXH5bNYBc
+ rm /tmp/tmp.uhMuNTNDpk /tmp/tmp.jwXH5bNYBc
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7fg0xWphqv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZMNAN7yjCQ
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.7fg0xWphqv
++ cat /tmp/tmp.ZMNAN7yjCQ
++ rm /tmp/tmp.7fg0xWphqv /tmp/tmp.ZMNAN7yjCQ
++ return 0
+ wait_pod postgres-operator-6f57774f4d-mdqgz
+ local pod=postgres-operator-6f57774f4d-mdqgz
+ local ns=
+ set +o xtrace
postgres-operator-6f57774f4d-mdqgztrue
+ [[ -n '' ]]
+ kubectl_bin diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.h5nywZTNA1
++ mktemp
+ local LAST_ERR=/tmp/tmp.Q7fqQmNN4M
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.h5nywZTNA1
+ cat /tmp/tmp.Q7fqQmNN4M
+ rm /tmp/tmp.h5nywZTNA1 /tmp/tmp.Q7fqQmNN4M
+ return 0
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo},
			{"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest},
			{"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger},
			{"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer},
			{"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha},
			{"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.r9jCK42Ebh
++ mktemp
+ local LAST_ERR=/tmp/tmp.pY19B8djrq
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo},
			{"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest},
			{"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger},
			{"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer},
			{"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha},
			{"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.r9jCK42Ebh
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.pY19B8djrq
+ rm /tmp/tmp.r9jCK42Ebh /tmp/tmp.pY19B8djrq
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OG1gzMPjUb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ZHl10C4SAD
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.OG1gzMPjUb
++ cat /tmp/tmp.ZHl10C4SAD
++ rm /tmp/tmp.OG1gzMPjUb /tmp/tmp.ZHl10C4SAD
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.AFYqOlgPg9
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3D4gG5NcOr
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.AFYqOlgPg9
+++ cat /tmp/tmp.3D4gG5NcOr
+++ rm /tmp/tmp.AFYqOlgPg9 /tmp/tmp.3D4gG5NcOr
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name
+ compare_generation 3 deployment some-name-pgbouncer
+ local generation=3
+ local resource=deployment
+ local name=some-name-pgbouncer
+ local current_generation
++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.vim58tsnPC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.fuKndXQEME
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.vim58tsnPC
++ cat /tmp/tmp.fuKndXQEME
++ rm /tmp/tmp.vim58tsnPC /tmp/tmp.fuKndXQEME
++ return 0
+ current_generation=2
+ set +o xtrace
......................................................................................++ kubectl_bin get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qtubUhF1sC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nARKvXmmV6
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.qtubUhF1sC
++ cat /tmp/tmp.nARKvXmmV6
++ rm /tmp/tmp.qtubUhF1sC /tmp/tmp.nARKvXmmV6
++ return 0
+ [[ 1.6.0 != 1.6.0 ]]
++ kubectl_bin get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.HDPcCxQCCb
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8puYxHKst4
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.HDPcCxQCCb
++ cat /tmp/tmp.8puYxHKst4
++ rm /tmp/tmp.HDPcCxQCCb /tmp/tmp.8puYxHKst4
++ return 0
+ [[ 1.6.0 != 1.6.0 ]]
+ sleep 10
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Am2Qv439FA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zrpgU5aQ0h
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Am2Qv439FA
++ cat /tmp/tmp.zrpgU5aQ0h
++ rm /tmp/tmp.Am2Qv439FA /tmp/tmp.zrpgU5aQ0h
++ return 0
+ local client_container=pg-client-76df9df876-hkdkb
+ kubectl_bin exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1caD81PQfE
++ mktemp
+ local LAST_ERR=/tmp/tmp.tZnoo5Dykx
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.1caD81PQfE
+ cat /tmp/tmp.tZnoo5Dykx
+ rm /tmp/tmp.1caD81PQfE /tmp/tmp.tZnoo5Dykx
+ return 0
+ '[' '!' -s /tmp/tmp.wKxCFsu2FT/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.wKxCFsu2FT/select-1.sql
+ backup_time_interval_min=5
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]},
			{"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}}
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.q2bqAupiZR
++ mktemp
+ local LAST_ERR=/tmp/tmp.dP9CmobiJe
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]},
			{"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}}
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.q2bqAupiZR
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.dP9CmobiJe
+ rm /tmp/tmp.q2bqAupiZR /tmp/tmp.dP9CmobiJe
+ return 0
+ sleep 10
++ next_backup_after_sec some-name-5min-backup
++ local conf_map_name=some-name-5min-backup
++++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.AlLOwW6WKK
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.9xJWepzlkP
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.AlLOwW6WKK
++++ cat /tmp/tmp.9xJWepzlkP
++++ rm /tmp/tmp.AlLOwW6WKK /tmp/tmp.9xJWepzlkP
++++ return 0
+++ /usr/bin/date +%s --date=2024-04-24T16:42:56Z --utc
++ local configmap_creation_time=1713976976
+++ grep -Eo '[0-9]+'
+++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}'
+++ jq .schedule
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fC5LtL06Zk
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XVJTE9hn0u
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.fC5LtL06Zk
+++ cat /tmp/tmp.XVJTE9hn0u
+++ rm /tmp/tmp.fC5LtL06Zk /tmp/tmp.XVJTE9hn0u
+++ return 0
++ local backup_time_interval_sec=300
+++ calculate_next_backup_start 1713976976 300
+++ local start_time_sec=1713976976
+++ local backup_time_interval_sec=300
+++ local next_interval_hop=1713977276
+++ local next_interval_min_edge=176
+++ local next_backup_at_sec=1713977100
+++ echo 1713977100
+++ get_current_epoch_time
++++ grep date
++++ curl -sI https://percona.com/
++++ /usr/bin/sed 's/date: //g'
+++ local 'curent_date=Wed, 24 Apr 2024 16:43:01 GMT
'
++++ /usr/bin/date --date 'Wed, 24 Apr 2024 16:43:01 GMT
' --utc +%s
+++ echo 1713976981
++ local result=119
++ [[ 119 -gt 0 ]]
++ echo 119
+ sleep 119
+ kubectl_bin wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s
++ mktemp
+ local LAST_OUT=/tmp/tmp.EOuVUJPh7Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.5wAOTydaCu
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.EOuVUJPh7Y
job.batch/some-name-full-sch-backup condition met
+ cat /tmp/tmp.5wAOTydaCu
+ rm /tmp/tmp.EOuVUJPh7Y /tmp/tmp.5wAOTydaCu
+ return 0
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Yo6Ldm6FBZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QriX915K4y
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl -n upgrade-4817 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Yo6Ldm6FBZ
++ cat /tmp/tmp.QriX915K4y
++ rm /tmp/tmp.Yo6Ldm6FBZ /tmp/tmp.QriX915K4y
++ return 0
+ local client_container=pg-client-76df9df876-hkdkb
+ kubectl_bin exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.kxfRIYBpkY
++ mktemp
+ local LAST_ERR=/tmp/tmp.JsrT6EbJW6
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-76df9df876-hkdkb -n upgrade-4817 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.kxfRIYBpkY
+ cat /tmp/tmp.JsrT6EbJW6
+ rm /tmp/tmp.kxfRIYBpkY /tmp/tmp.JsrT6EbJW6
+ return 0
+ '[' '!' -s /tmp/tmp.wKxCFsu2FT/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.wKxCFsu2FT/select-1.sql
+ kubectl create secret generic some-name-ssl-ca --from-file=ca.crt=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/ca.crt
secret/some-name-ssl-ca created
+ kubectl create secret tls some-name-ssl-keypair --cert=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/server.crt --key=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/server.key
secret/some-name-ssl-keypair created
+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.HT2yugIQVc
++ mktemp
+ local LAST_ERR=/tmp/tmp.93ltQZPTwa
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.HT2yugIQVc
deployment.apps "pg-client" deleted
+ cat /tmp/tmp.93ltQZPTwa
+ rm /tmp/tmp.HT2yugIQVc /tmp/tmp.93ltQZPTwa
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A4hZ4ebiH1
++ mktemp
+ local LAST_ERR=/tmp/tmp.KIxbSlCjji
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.A4hZ4ebiH1
deployment.apps/pg-client created
+ cat /tmp/tmp.KIxbSlCjji
+ rm /tmp/tmp.A4hZ4ebiH1 /tmp/tmp.KIxbSlCjji
+ return 0
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/tlsOnly","value":true},
			{"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''},
			{"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''},
		]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.h9XlkCGTGQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.GLTOQO1ccf
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json -p '[
			{"op":"add","path":"/spec/tlsOnly","value":true},
			{"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''},
			{"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''},
			{"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''},
		]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.h9XlkCGTGQ
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.GLTOQO1ccf
+ rm /tmp/tmp.h9XlkCGTGQ /tmp/tmp.GLTOQO1ccf
+ return 0
+ disable_pgBouncer some-name
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ljYJ73UOu0
++ mktemp
+ local LAST_ERR=/tmp/tmp.dAZcsCXCzT
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ljYJ73UOu0
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.dAZcsCXCzT
+ rm /tmp/tmp.ljYJ73UOu0 /tmp/tmp.dAZcsCXCzT
+ return 0
+ kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.zfdUcwz3bV
++ mktemp
+ local LAST_ERR=/tmp/tmp.qrrBO9Wk99
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.zfdUcwz3bV
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.qrrBO9Wk99
+ rm /tmp/tmp.zfdUcwz3bV /tmp/tmp.qrrBO9Wk99
+ return 0
+ enable_pgBouncer some-name
+ local cluster_name=some-name
+ local replicas_num=1
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ysYGJiNYTa
++ mktemp
+ local LAST_ERR=/tmp/tmp.HOoOC2QQKP
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ysYGJiNYTa
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.HOoOC2QQKP
+ rm /tmp/tmp.ysYGJiNYTa /tmp/tmp.HOoOC2QQKP
+ return 0
+ sleep 60
+ kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.RdMCUvob0D
++ mktemp
+ local LAST_ERR=/tmp/tmp.11OwdYGlUF
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.RdMCUvob0D
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.11OwdYGlUF
+ rm /tmp/tmp.RdMCUvob0D /tmp/tmp.11OwdYGlUF
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=upgrade-4817
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9Hg92wx6gF
+++ mktemp
++ local LAST_ERR=/tmp/tmp.G536fifjrI
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.9Hg92wx6gF
++ cat /tmp/tmp.G536fifjrI
++ rm /tmp/tmp.9Hg92wx6gF /tmp/tmp.G536fifjrI
++ return 0
+ status_query='kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wnmIRyvyGj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZrcH8NpsXR
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+++ kubectl -n upgrade-4817 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.wnmIRyvyGj
+++ cat /tmp/tmp.ZrcH8NpsXR
+++ rm /tmp/tmp.wnmIRyvyGj /tmp/tmp.ZrcH8NpsXR
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=upgrade-4817
+ sleep 10
+ set +o xtrace
some-name-pgbouncer
+ sleep 10
+ compare_psql select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817 verify-ca
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local sslmode=verify-ca
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql
+ run_psql_ssl '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-4817 postgres '' verify-ca
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-4817
+ local driver=postgres
++ kubectl_bin get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.DkxQSU5Stf
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i3TIajckET
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.DkxQSU5Stf
++ cat /tmp/tmp.i3TIajckET
++ rm /tmp/tmp.DkxQSU5Stf /tmp/tmp.i3TIajckET
++ return 0
+ local client_container=pg-client-7f96cb6f54-d4nx5
+ local sslmode=verify-ca
+ kubectl_bin exec pg-client-7f96cb6f54-d4nx5 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817'
++ mktemp
+ local LAST_OUT=/tmp/tmp.retdUoN5V7
++ mktemp
+ local LAST_ERR=/tmp/tmp.60yXt29NVC
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl exec pg-client-7f96cb6f54-d4nx5 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-4817'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.retdUoN5V7
+ cat /tmp/tmp.60yXt29NVC
+ rm /tmp/tmp.retdUoN5V7 /tmp/tmp.60yXt29NVC
+ return 0
+ '[' '!' -s /tmp/tmp.wKxCFsu2FT/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-723/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.wKxCFsu2FT/select-1.sql
+ destroy upgrade-4817
+ grep -v level=info
++ get_operator_pod
+ grep -v level=debug
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+ grep -v 'Getting pods from source'
+++ mktemp
+ grep -v 'Getting tasks for pod'
+ grep -v 'the object has been modified'
+ grep -v 'get backup status: Job.batch'
++ local LAST_OUT=/tmp/tmp.LJWHa4Ymx6
+ tee /tmp/tmp.wKxCFsu2FT/operator.log
+ sort -u
+++ mktemp
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
++ local LAST_ERR=/tmp/tmp.tpqsSYv705
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.LJWHa4Ymx6
++ cat /tmp/tmp.tpqsSYv705
++ rm /tmp/tmp.LJWHa4Ymx6 /tmp/tmp.tpqsSYv705
++ return 0
+ kubectl_bin logs postgres-operator-6f57774f4d-mdqgz -c operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.YQGW93Q8tB
++ mktemp
+ local LAST_ERR=/tmp/tmp.PSEZ6yA0n2
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl logs postgres-operator-6f57774f4d-mdqgz -c operator
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.YQGW93Q8tB
+ cat /tmp/tmp.PSEZ6yA0n2
+ rm /tmp/tmp.YQGW93Q8tB /tmp/tmp.PSEZ6yA0n2
+ return 0

  
    
      
          
            
                
                  
                    
                      
                        
                            
                                        
                                            
                                            ]
                                        {
                                        }
                                        },
                                    ]
                                },
                            }
                          }
                        {
                        }
                        },
                      ]
                      {
                      }
                      },
                    ]
                    ],
                    {
                    }
                    },
                    }, {
                    },{
                    }]
                    }],
                  {
                  }
                  },
                ,
                ],
                }
                },
                }],
            }
            },
        }
        },
      {
      }
    ],
    }
    },
  }
  },
 },
{
}
},
2024/04/24 16:42:31 INF    1 (localhost:4150) connecting to nsqd
2024/04/24 16:45:00 INF    2 (localhost:4150) connecting to nsqd
2024/04/24 16:45:32 INF    3 (localhost:4150) connecting to nsqd
2024/04/24 16:45:56 INF    4 (localhost:4150) connecting to nsqd
2024/04/24 16:47:47 INF    5 (localhost:4150) connecting to nsqd
2024/04/24 16:48:32 INF    6 (localhost:4150) connecting to nsqd
2024/04/24 16:48:38 INF    7 (localhost:4150) connecting to nsqd
2024/04/24 16:48:54 INF    8 (localhost:4150) connecting to nsqd
                "affinity": 
                "affinity": {                 "podAntiAffinity": {
                      "allowPrivilegeEscalation": false,
                "annotations": {"keep-backups":"false","keep-data":"false"},
    "apiVersion": "apps/v1",
    "apiVersion": "batch/v1",
  "apiVersion": "v1",
                    "backrest-command": "backup",
                          "configMap": {
                        "containerPort": 5432,
                "containers": [{
    "cpu": "1",
                                                "crunchydata"
                    "crunchy-pgbouncer": "true",
                "crunchy-pgbouncer": "true",
            "crunchy-pgbouncer": "true",
                            "defaultMode": 288
                    "defaultMode": 288,
                "dnsPolicy": "ClusterFirst"
                      "emptyDir": {
                    "emptyDir": {
                    "env": [{
                                "fieldPath": "metadata.namespace"
                            "fieldRef": {
                  "fsGroup": 2,
I0424 16:38:43.955631       1 shared_informer.go:240] Waiting for caches to sync for namespace
I0424 16:38:43.956072       1 shared_informer.go:247] Caches are synced for namespace 
I0424 16:38:44.165443       1 shared_informer.go:240] Waiting for caches to sync for upgrade-4817
I0424 16:38:44.266114       1 shared_informer.go:247] Caches are synced for upgrade-4817 
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer",
                    "imagePullPolicy": "IfNotPresent",
                                            "key": "crunchy-pgbouncer",
                                "key": "password"
                                            "key": "pg-cluster",
                                            "key": "pgo-backrest-job",
                                            "key": "pgo-pg-database",
                                            "key": "pg-pod-anti-affinity",
                                            "key": "vendor",
    "kind": "Deployment",
    "kind": "Job",
  "kind": "Service",
                "labels": {
        "labels": {
      "labels": {
                                "labelSelector": {
                                    "matchExpressions": [
            "matchLabels": {
                "maxSurge": 1
                "maxUnavailable": 1,
                        "medium": "Memory",
                      "medium": "Memory",
    "memory": "128Mi"
            "metadata": {
    "metadata": {
  "metadata": {
                        "mountPath": "/pgconf/",
                        "mountPath": "/pgconf/tls/pgbouncer",
                            "mountPath": "/tmp",
                          "mountPath": "/tmp",
                    "name": "backrest",
                        "name": "COMMAND",
                        "name": "COMMAND_OPTS",
                        "name": "MODE",
                        "name": "NAMESPACE",
                        "name": "PGBACKREST_DB_PATH",
                        "name": "PGBACKREST_LOG_PATH",
                        "name": "PGBACKREST_REPO1_PATH",
                        "name": "PGBACKREST_REPO1_TYPE",
                        "name": "PGBACKREST_STANZA",
                        "name": "pgbouncer"
                    "name": "pgbouncer",
                        "name": "pgbouncer-conf",
                    "name": "pgbouncer-conf",
                        "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
                        "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
                        "name": "PGHA_PGBACKREST_S3_VERIFY_TLS",
                        "name": "PG_PASSWORD",
                        "name": "PG_PRIMARY_SERVICE_NAME",
                        "name": "PITR_TARGET",
                        "name": "PODNAME",
        "name": "postgres",
                "name": "some-name-full-sch-backup",
                    "name": "some-name-pgbouncer",
                "name": "some-name-pgbouncer",
            "name": "some-name-pgbouncer",
          "name": "some-name-pgbouncer"
        "name": "some-name-pgbouncer",
      "name": "some-name-pgbouncer",
                            "name": "some-name-pgbouncer-cm"
                                "name": "some-name-pgbouncer-secret",
                            "name": "some-name-pgbouncer-secret",
                              "name": "some-name-ssl-ca"
                              "name": "some-name-ssl-keypair"
                        "name": "tls-pgbouncer"
                    "name": "tls-pgbouncer",
                            "name": "tmp"
                          "name": "tmp"
                      "name": "tmp",
                    "name": "tmp",
        "nodePort": 0
                                            "operator": "Exists"
                                            "operator": "In",
                    "pg-cluster": "some-name"
                    "pg-cluster": "some-name",
                "pg-cluster": "some-name",
            "pg-cluster": "some-name",
          "pg-cluster": "some-name",
                    "pgo-backrest-job": "true",
                    "pgo-backrest": "true",
                    "pg-pod-anti-affinity": "preferred",
                "pg-pod-anti-affinity": "preferred",
                            "podAffinityTerm": {
                                        "podAntiAffinity": {
                "podAntiAffinity": {
        "port": 5432,
                    "ports": [{
    "ports": [
                    "preferredDuringSchedulingIgnoredDuringExecution": [
                      "privileged": false,
                    "projected": {
                        "protocol": "TCP",
        "protocol": "TCP",
                        "readOnly": false
                      "readOnlyRootFilesystem": true
        "replicas": 1,
  "requests": {
"resources": {
                "restartPolicy": "Always",
                "restartPolicy": "Never"
            "rollingUpdate": {
                  "runAsNonRoot": true
                          "secret": {
                            "secretKeyRef": {
                    "securityContext": {
                "securityContext": {
                "securityContext": {},
        "selector": {
    "selector": {
                "serviceAccountName": "pgo-backrest",
                "serviceAccountName": "pgo-default",
                    "service-name": "some-name-pgbouncer",
                "service-name": "some-name-pgbouncer",
            "service-name": "some-name-pgbouncer",
      "service-name": "some-name-pgbouncer"
    "sessionAffinity": "None"
                        "sizeLimit": "16Mi"
                      "sizeLimit": "1Mi"
                                                "some-name"
                      "sources": [
            "spec": {
    "spec": {
  "spec": {
        "strategy": {
        "targetPort": 5432,
        "template": {
time="2024-04-24T16:38:43Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0
time="2024-04-24T16:38:46Z" level=error msg="create pgcluster resource: create pgcluster resource: pgclusters.pg.percona.com \"some-name\" already exists" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgc.(*Controller).onAdd()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgc/pgc.go:174" version=1.6.0
time="2024-04-24T16:41:25Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-24T16:41:25Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name-repl2/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
time="2024-04-24T16:41:44Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0
time="2024-04-24T16:41:44Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-24T16:41:44Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0
time="2024-04-24T16:47:46Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-24T16:47:46Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
                                "topologyKey": "kubernetes.io/hostname"
    "type": "ClusterIP",
            "type": "RollingUpdate",
                        "value": ""
                        "value": "backup"
                        "value": "false"
                        "valueFrom": {
                        "value": "pgbackrest"
                        "value": "posix"
                                            "values": [
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-k646g"
                        "value": "--stanza=db --type=full  --repo1-retention-full=3 --db-host=10.2.129.10 --db-path=/pgdata/some-name"
                        "value": "/tmp"
                    "vendor": "crunchydata"
                    "vendor": "crunchydata",
                "vendor": "crunchydata"
            "vendor": "crunchydata"
          "vendor": "crunchydata",
                    "volumeMounts": [
                "volumes": [
                            "weight": 1,
+ destroy_operator
+ kubectl_bin delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.HtCh3KKkUN
++ mktemp
+ local LAST_ERR=/tmp/tmp.hnrNIowYke
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.HtCh3KKkUN
+ cat /tmp/tmp.hnrNIowYke
Error from server (NotFound): error when deleting "/tmp/tmp.wKxCFsu2FT/operator.yaml": serviceaccounts "pgo-deployer-sa" not found
Error from server (NotFound): error when deleting "/tmp/tmp.wKxCFsu2FT/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): error when deleting "/tmp/tmp.wKxCFsu2FT/operator.yaml": configmaps "pgo-deployer-cm" not found
Error from server (NotFound): error when deleting "/tmp/tmp.wKxCFsu2FT/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found
Error from server (NotFound): error when deleting "/tmp/tmp.wKxCFsu2FT/operator.yaml": jobs.batch "pgo-deploy" not found
+ rm /tmp/tmp.HtCh3KKkUN /tmp/tmp.hnrNIowYke
+ return 1
+ true
+ yq w -d4 /tmp/tmp.wKxCFsu2FT/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall
+ kubectl_bin apply -f /tmp/tmp.wKxCFsu2FT/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.8ezeNDoxNu
++ mktemp
+ local LAST_ERR=/tmp/tmp.ESBlhpVIJM
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl apply -f /tmp/tmp.wKxCFsu2FT/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.8ezeNDoxNu
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.ESBlhpVIJM
+ rm /tmp/tmp.8ezeNDoxNu /tmp/tmp.ESBlhpVIJM
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=upgrade-4817
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n upgrade-4817 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.wKxCFsu2FT/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.6mCnGOUQrL
++ mktemp
+ local LAST_ERR=/tmp/tmp.ygQ2592s97
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete -f /tmp/tmp.wKxCFsu2FT/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6mCnGOUQrL
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.ygQ2592s97
+ rm /tmp/tmp.6mCnGOUQrL /tmp/tmp.ygQ2592s97
+ return 0
+ kubectl_bin get ns
+ awk '{print$1}'
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qac5T6oqLL
+ xargs kubectl delete ns --grace-period=0 --force=true
++ mktemp
+ local LAST_ERR=/tmp/tmp.YRIrg54tdK
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl get ns
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.qac5T6oqLL
+ cat /tmp/tmp.YRIrg54tdK
+ rm /tmp/tmp.qac5T6oqLL /tmp/tmp.YRIrg54tdK
+ return 0
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
namespace "upgrade-4817" force deleted
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.l0U9luJgbn
++ mktemp
+ local LAST_ERR=/tmp/tmp.p0pwb9jBDD
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.l0U9luJgbn
+ cat /tmp/tmp.p0pwb9jBDD
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.l0U9luJgbn /tmp/tmp.p0pwb9jBDD
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.cwAziYtp4z
++ mktemp
+ local LAST_ERR=/tmp/tmp.23yPb7WCnZ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-723-7ed19e708-2-upgrade
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.cwAziYtp4z
+ cat /tmp/tmp.23yPb7WCnZ
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.cwAziYtp4z /tmp/tmp.23yPb7WCnZ
+ return 1
+ true
+ rm -rf /tmp/tmp.wKxCFsu2FT