++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/logs/upgrade.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/logs/upgrade.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.14.4 ++ '[' v3 == v2 ']' + main + create_namespace upgrade-5454 + local namespace=upgrade-5454 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' + cleanup_rbac + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.5yjJIviMQv + local LAST_OUT=/tmp/tmp.DQziYFdCv6 ++ mktemp ++ mktemp + xargs kubectl delete ns + local LAST_ERR=/tmp/tmp.2WnrC0cGh9 + local exit_status=0 + local timeout=4 + set +o errexit + local LAST_ERR=/tmp/tmp.rHen99qTfv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl get ns + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5yjJIviMQv + cat /tmp/tmp.2WnrC0cGh9 + rm /tmp/tmp.5yjJIviMQv /tmp/tmp.2WnrC0cGh9 + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level namespace "upgrade-20793" deleted + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.DQziYFdCv6 + cat /tmp/tmp.rHen99qTfv Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.DQziYFdCv6 /tmp/tmp.rHen99qTfv + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.2JXcMr0WDL ++ mktemp + local LAST_ERR=/tmp/tmp.ZUYbaLQEQg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.2JXcMr0WDL + cat /tmp/tmp.ZUYbaLQEQg Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.2JXcMr0WDL /tmp/tmp.ZUYbaLQEQg + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace upgrade-5454 ++ mktemp + local LAST_OUT=/tmp/tmp.GOPcLI4pai ++ mktemp + local LAST_ERR=/tmp/tmp.gr0OcRr5tQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete namespace upgrade-5454 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete namespace upgrade-5454 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete namespace upgrade-5454 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.GOPcLI4pai + cat /tmp/tmp.gr0OcRr5tQ Error from server (NotFound): namespaces "upgrade-5454" not found + rm /tmp/tmp.GOPcLI4pai /tmp/tmp.gr0OcRr5tQ + return 1 + : + wait_for_delete namespace/upgrade-5454 + local res=namespace/upgrade-5454 + set +o xtrace namespace/upgrade-5454 - Error from server (NotFound): namespaces "upgrade-5454" not found + kubectl_bin create namespace upgrade-5454 ++ mktemp + local LAST_OUT=/tmp/tmp.5fmAHfjNFW ++ mktemp + local LAST_ERR=/tmp/tmp.ozvyUm1wd9 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl create namespace upgrade-5454 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5fmAHfjNFW namespace/upgrade-5454 created + cat /tmp/tmp.ozvyUm1wd9 + rm /tmp/tmp.5fmAHfjNFW /tmp/tmp.ozvyUm1wd9 + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.k9cmaPY18q +++ mktemp ++ local LAST_ERR=/tmp/tmp.oPYn2qpILE ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.k9cmaPY18q ++ cat /tmp/tmp.oPYn2qpILE ++ rm /tmp/tmp.k9cmaPY18q /tmp/tmp.oPYn2qpILE ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-757-14c649e4c-3-upgrade --namespace=upgrade-5454 ++ mktemp + local LAST_OUT=/tmp/tmp.jSxFd5kMRW ++ mktemp + local LAST_ERR=/tmp/tmp.rK7D0xqVb8 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-757-14c649e4c-3-upgrade --namespace=upgrade-5454 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jSxFd5kMRW Context "gke_cloud-dev-112233_us-central1-c_jen-pg-757-14c649e4c-3-upgrade" modified. + cat /tmp/tmp.rK7D0xqVb8 + rm /tmp/tmp.jSxFd5kMRW /tmp/tmp.rK7D0xqVb8 + return 0 + cluster=some-name + operator_action=install + create_or_upgrade_to_version some-name 1.5.0 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install + local cluster=some-name + local pgo_tag=1.5.0 + local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml + local namespace_mode=dynamic + local operator_action=install + IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.0-postgres-operator + IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.0-pgo-deployer + IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer + IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha + IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest + IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo + IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger + [[ xinstall == \x\i\n\s\t\a\l\l ]] + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml + local namespace_mode=dynamic + local action=install + local namespace=upgrade-5454 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml dynamic install upgrade-5454 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml + local namespace_mode=dynamic + local operator_action=install + local namespace=upgrade-5454 + local dont_send_telemetry=false + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-5454"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + cat /dev/fd/63 - + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-5454"#g' ++ echo 'crunchy_debug: "true"' ++ echo percona/percona-postgresql-operator:1.5.0 ++ cut -d: -f1 ++ echo percona/percona-postgresql-operator:1.5.0 ++ cut -d: -f2 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g' + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.0"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.4.0_operator.yaml metadata.namespace upgrade-5454 + yq w -d3 - 'subjects[0].namespace' upgrade-5454 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install ++ cat /tmp/tmp.RFCZbnUcQr/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-5454" namespace_mode: "dynamic" pgo_image_prefix: "percona/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "1.5.0" pgo_operator_namespace: "upgrade-5454"' + yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.0-pgo-deployer + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.IqxBjCX1Dh ++ mktemp + local LAST_ERR=/tmp/tmp.Q1tNHoJikV + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.IqxBjCX1Dh serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.Q1tNHoJikV + rm /tmp/tmp.IqxBjCX1Dh /tmp/tmp.Q1tNHoJikV + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-5454 + retry=0 + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ybJm2AWSy8 ++ mktemp + local LAST_ERR=/tmp/tmp.TyMnCfvhl9 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ybJm2AWSy8 serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.TyMnCfvhl9 + rm /tmp/tmp.ybJm2AWSy8 /tmp/tmp.TyMnCfvhl9 + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1ULry7aRtu +++ mktemp ++ local LAST_ERR=/tmp/tmp.D27scWLVaB ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.1ULry7aRtu ++ cat /tmp/tmp.D27scWLVaB ++ rm /tmp/tmp.1ULry7aRtu /tmp/tmp.D27scWLVaB ++ return 0 + wait_pod postgres-operator-7f45dc6878-j9x7b + local pod=postgres-operator-7f45dc6878-j9x7b + local ns= + set +o xtrace postgres-operator-7f45dc6878-j9x7btrue + [[ -n '' ]] ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.7Wg6zIzER4 +++ mktemp ++ local LAST_ERR=/tmp/tmp.3xQ7jBI245 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 0 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 4 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 8 ++ set -o errexit ++ cat /tmp/tmp.7Wg6zIzER4 ++ cat /tmp/tmp.3xQ7jBI245 Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found ++ rm /tmp/tmp.7Wg6zIzER4 /tmp/tmp.3xQ7jBI245 ++ return 1 + [[ x == \x ]] + desc 'start cluster' + set +o xtrace ----------------------------------------------------------------------------------- start cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name + local cluster=some-name + local namespace=upgrade-5454 + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/users-secret.yaml + kubectl_bin -n upgrade-5454 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.MzYeQ6O1H6 ++ mktemp + local LAST_ERR=/tmp/tmp.kd7rcSfeQQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.MzYeQ6O1H6 secret/some-name-users-secret created + cat /tmp/tmp.kd7rcSfeQQ + rm /tmp/tmp.MzYeQ6O1H6 /tmp/tmp.kd7rcSfeQQ + return 0 + yq w - metadata.name some-name-pmm-secret + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/users-secret.yaml + kubectl_bin -n upgrade-5454 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xkuxvuF746 ++ mktemp + local LAST_ERR=/tmp/tmp.K6VvU6BQ0K + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xkuxvuF746 secret/some-name-pmm-secret created + cat /tmp/tmp.K6VvU6BQ0K + rm /tmp/tmp.xkuxvuF746 /tmp/tmp.K6VvU6BQ0K + return 0 + spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_cr.yaml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_cr.yaml + local backup=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/conf/client.yml + local namespace=upgrade-5454 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name upgrade-5454 + local cluster=some-name + local namespace=upgrade-5454 + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/users-secret.yaml + kubectl_bin -n upgrade-5454 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.wiloN0cuNu ++ mktemp + local LAST_ERR=/tmp/tmp.MJh7xYd1Li + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.wiloN0cuNu secret/some-name-users-secret configured + cat /tmp/tmp.MJh7xYd1Li + rm /tmp/tmp.wiloN0cuNu /tmp/tmp.MJh7xYd1Li + return 0 + yq w - metadata.name some-name-pmm-secret + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/users-secret.yaml + kubectl_bin -n upgrade-5454 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.yjYzvlW4UU ++ mktemp + local LAST_ERR=/tmp/tmp.3BCcnvv83H + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yjYzvlW4UU secret/some-name-pmm-secret configured + cat /tmp/tmp.3BCcnvv83H + rm /tmp/tmp.yjYzvlW4UU /tmp/tmp.3BCcnvv83H + return 0 + kubectl_bin -n upgrade-5454 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.0nUdhoOn6b ++ mktemp + local LAST_ERR=/tmp/tmp.jYWVgKsd86 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.0nUdhoOn6b deployment.apps/pg-client created + cat /tmp/tmp.jYWVgKsd86 + rm /tmp/tmp.0nUdhoOn6b /tmp/tmp.jYWVgKsd86 + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_cr.yaml some-name false false false false false false false false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_cr.yaml metadata.annotations.current-primary some-name + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - metadata.labels.name some-name + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.pgBouncer.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbouncer + yq w - spec.clustername some-name + yq w - spec.database some-name + yq w - spec.keepBackups false + yq w - spec.name some-name + yq w - spec.keepData false + yq w - spec.pgPrimary.image percona/percona-postgresql-operator:1.5.0-ppg14-postgres-ha + yq w - spec.backup.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest + yq w - spec.secretsName some-name-users-secret + yq w - spec.pmm.image perconalab/pmm-client:2.41.0 + yq w - spec.pgBadger.image percona/percona-postgresql-operator:1.5.0-ppg14-pgbadger + yq w - spec.user some-name + yq w - metadata.labels.deployment-name some-name + yq w - spec.backup.backrestRepoImage percona/percona-postgresql-operator:1.5.0-ppg14-pgbackrest-repo + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.RFCZbnUcQr/cr.yaml -n upgrade-5454 ++ mktemp + local LAST_OUT=/tmp/tmp.HE3gNjydnP ++ mktemp + local LAST_ERR=/tmp/tmp.sLb5XZc5ps + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl apply -f /tmp/tmp.RFCZbnUcQr/cr.yaml -n upgrade-5454 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HE3gNjydnP perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.sLb5XZc5ps + rm /tmp/tmp.HE3gNjydnP /tmp/tmp.sLb5XZc5ps + return 0 + wait_deployment some-name-backrest-shared-repo + local name=some-name-backrest-shared-repo + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name-backrest-shared-repo.. + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name........ + [[ false == \g\c\s\+* ]] + wait_job_completion some-name-stanza-create false + local job=some-name-stanza-create + local mandatory=false + local ns=upgrade-5454 + retry=0 + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' some-name-stanza-create+ retry=0 ++ kubectl_bin -n upgrade-5454 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name + local job=backrest-backup-some-name + local mandatory=true + local ns=upgrade-5454 + retry=0 + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' backrest-backup-some-name+ retry=0 ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.URHEn1eZ1A +++ mktemp ++ local LAST_ERR=/tmp/tmp.PnM2TuSsVl ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.URHEn1eZ1A ++ cat /tmp/tmp.PnM2TuSsVl ++ rm /tmp/tmp.URHEn1eZ1A /tmp/tmp.PnM2TuSsVl ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7Wx78lRvPo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8wzGcYAgd7 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7Wx78lRvPo +++ cat /tmp/tmp.8wzGcYAgd7 +++ rm /tmp/tmp.7Wx78lRvPo /tmp/tmp.8wzGcYAgd7 +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-5454 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c8NK9ciD9R ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hvbEIx0nIn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.c8NK9ciD9R +++ cat /tmp/tmp.hvbEIx0nIn +++ rm /tmp/tmp.c8NK9ciD9R /tmp/tmp.hvbEIx0nIn +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.flEcXPPaqM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Qh7gLCkyyr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.flEcXPPaqM +++ cat /tmp/tmp.Qh7gLCkyyr +++ rm /tmp/tmp.flEcXPPaqM /tmp/tmp.Qh7gLCkyyr +++ return 0 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4gavkusTYe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zy7q5fMPVL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.4gavkusTYe +++ cat /tmp/tmp.zy7q5fMPVL +++ rm /tmp/tmp.4gavkusTYe /tmp/tmp.zy7q5fMPVL +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.G5K0APyfVG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.SfeXR3ZTJe ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.G5K0APyfVG ++++ cat /tmp/tmp.SfeXR3ZTJe ++++ rm /tmp/tmp.G5K0APyfVG /tmp/tmp.SfeXR3ZTJe ++++ return 0 +++ kubectl_bin -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Qf3sRd66xa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Si8ZJMP5Ob +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Qf3sRd66xa +++ cat /tmp/tmp.Si8ZJMP5Ob +++ rm /tmp/tmp.Qf3sRd66xa /tmp/tmp.Si8ZJMP5Ob +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.upgrade-5454 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name.upgrade-5454 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RqchoPTNO3 +++ mktemp ++ local LAST_ERR=/tmp/tmp.5RykvcaBQh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.RqchoPTNO3 ++ cat /tmp/tmp.5RykvcaBQh ++ rm /tmp/tmp.RqchoPTNO3 /tmp/tmp.5RykvcaBQh ++ return 0 + local client_container=pg-client-76df9df876-hn2r6 + kubectl_bin exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.V4QBL9Z07z ++ mktemp + local LAST_ERR=/tmp/tmp.PUkLp2TSGl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.V4QBL9Z07z + cat /tmp/tmp.PUkLp2TSGl + rm /tmp/tmp.V4QBL9Z07z /tmp/tmp.PUkLp2TSGl + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-5454 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WIoJSOC7P8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.95ozLyekqi +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.WIoJSOC7P8 +++ cat /tmp/tmp.95ozLyekqi +++ rm /tmp/tmp.WIoJSOC7P8 /tmp/tmp.95ozLyekqi +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A1xRST5PrQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0lXYECkPqc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.A1xRST5PrQ +++ cat /tmp/tmp.0lXYECkPqc +++ rm /tmp/tmp.A1xRST5PrQ /tmp/tmp.0lXYECkPqc +++ return 0 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RNj30iMRVx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1l8VSGk0zn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RNj30iMRVx +++ cat /tmp/tmp.1l8VSGk0zn +++ rm /tmp/tmp.RNj30iMRVx /tmp/tmp.1l8VSGk0zn +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.59NA6iiHkX +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.A6FNv18aci ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.59NA6iiHkX ++++ cat /tmp/tmp.A6FNv18aci ++++ rm /tmp/tmp.59NA6iiHkX /tmp/tmp.A6FNv18aci ++++ return 0 +++ kubectl_bin -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.AP4ZAWl05v ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ckNzA5Xndb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.AP4ZAWl05v +++ cat /tmp/tmp.ckNzA5Xndb +++ rm /tmp/tmp.AP4ZAWl05v /tmp/tmp.ckNzA5Xndb +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.upgrade-5454 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name.upgrade-5454 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jUdfExMV0X +++ mktemp ++ local LAST_ERR=/tmp/tmp.hRBwGxZWuD ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.jUdfExMV0X ++ cat /tmp/tmp.hRBwGxZWuD ++ rm /tmp/tmp.jUdfExMV0X /tmp/tmp.hRBwGxZWuD ++ return 0 + local client_container=pg-client-76df9df876-hn2r6 + kubectl_bin exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.9NO0gc0fF0 ++ mktemp + local LAST_ERR=/tmp/tmp.QHVfjZSwtJ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9NO0gc0fF0 + cat /tmp/tmp.QHVfjZSwtJ + rm /tmp/tmp.9NO0gc0fF0 /tmp/tmp.QHVfjZSwtJ + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-5454 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mZiYqBt7ub ++++ mktemp +++ local LAST_ERR=/tmp/tmp.leRLfRbrn8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mZiYqBt7ub +++ cat /tmp/tmp.leRLfRbrn8 +++ rm /tmp/tmp.mZiYqBt7ub /tmp/tmp.leRLfRbrn8 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c1EBdOeoij ++++ mktemp +++ local LAST_ERR=/tmp/tmp.53oyfBnnws +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.c1EBdOeoij +++ cat /tmp/tmp.53oyfBnnws +++ rm /tmp/tmp.c1EBdOeoij /tmp/tmp.53oyfBnnws +++ return 0 +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Pgs1gkXuJs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5trZjAsUOV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Pgs1gkXuJs +++ cat /tmp/tmp.5trZjAsUOV +++ rm /tmp/tmp.Pgs1gkXuJs /tmp/tmp.5trZjAsUOV +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.s1hqNSHdZ3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.0jLt62RrIK ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.s1hqNSHdZ3 ++++ cat /tmp/tmp.0jLt62RrIK ++++ rm /tmp/tmp.s1hqNSHdZ3 /tmp/tmp.0jLt62RrIK ++++ return 0 +++ kubectl_bin -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.joYV8xmwo4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nT72TMIhz2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.joYV8xmwo4 +++ cat /tmp/tmp.nT72TMIhz2 +++ rm /tmp/tmp.joYV8xmwo4 /tmp/tmp.nT72TMIhz2 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.upgrade-5454 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name.upgrade-5454 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AnlCfxkXF0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.RYUtW8xbHh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl -n upgrade-5454 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.AnlCfxkXF0 ++ cat /tmp/tmp.RYUtW8xbHh ++ rm /tmp/tmp.AnlCfxkXF0 /tmp/tmp.RYUtW8xbHh ++ return 0 + local client_container=pg-client-76df9df876-hn2r6 + kubectl_bin exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.nHhufyLIyW ++ mktemp + local LAST_ERR=/tmp/tmp.JZsrspvYg3 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl exec pg-client-76df9df876-hn2r6 -n upgrade-5454 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-5454.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.nHhufyLIyW + cat /tmp/tmp.JZsrspvYg3 + rm /tmp/tmp.nHhufyLIyW /tmp/tmp.JZsrspvYg3 + return 0 + sleep 10 + [[ xinstall == \x\u\p\d\a\t\e ]] + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ak88iiSe3x +++ mktemp ++ local LAST_ERR=/tmp/tmp.N75TXBoIsN ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Ak88iiSe3x ++ cat /tmp/tmp.N75TXBoIsN ++ rm /tmp/tmp.Ak88iiSe3x /tmp/tmp.N75TXBoIsN ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yavbbyr6hR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IO02wEeBlt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.yavbbyr6hR +++ cat /tmp/tmp.IO02wEeBlt +++ rm /tmp/tmp.yavbbyr6hR /tmp/tmp.IO02wEeBlt +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name + sleep 10 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.LoMENBhF2C ++ mktemp + local LAST_ERR=/tmp/tmp.xAUVfbUixh + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.LoMENBhF2C perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.xAUVfbUixh + rm /tmp/tmp.LoMENBhF2C /tmp/tmp.xAUVfbUixh + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Qjm58ikqgm +++ mktemp ++ local LAST_ERR=/tmp/tmp.IQjB707fxk ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Qjm58ikqgm ++ cat /tmp/tmp.IQjB707fxk ++ rm /tmp/tmp.Qjm58ikqgm /tmp/tmp.IQjB707fxk ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5Vgzd6k7V7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.c3x6jGKkMI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5Vgzd6k7V7 +++ cat /tmp/tmp.c3x6jGKkMI +++ rm /tmp/tmp.5Vgzd6k7V7 /tmp/tmp.c3x6jGKkMI +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name-pgbouncer + operator_action=update + create_or_upgrade_to_version some-name 1.5.1 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update + local cluster=some-name + local pgo_tag=1.5.1 + local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml + local namespace_mode=dynamic + local operator_action=update + IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.1-postgres-operator + IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.1-pgo-deployer + IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer + IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha + IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest + IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo + IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger + [[ xupdate == \x\i\n\s\t\a\l\l ]] + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml + local namespace_mode=dynamic + local action=update + local namespace=upgrade-5454 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml dynamic update upgrade-5454 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml + local namespace_mode=dynamic + local operator_action=update + local namespace=upgrade-5454 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-5454"#g' + cat /dev/fd/63 - + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-5454"#g' ++ echo 'crunchy_debug: "true"' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml 'data[values.yaml]' ++ echo percona/percona-postgresql-operator:1.5.1 ++ cut -d: -f1 ++ cut -d: -f2 ++ echo percona/percona-postgresql-operator:1.5.1 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g' + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.1"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/e2e-tests/upgrade/conf/1.5.0_operator.yaml metadata.namespace upgrade-5454 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update + yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.1-pgo-deployer + yq w -d3 - 'subjects[0].namespace' upgrade-5454 ++ cat /tmp/tmp.RFCZbnUcQr/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-5454" namespace_mode: "dynamic" pgo_image_prefix: "percona/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "1.5.1" pgo_operator_namespace: "upgrade-5454"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.bx4Lfln2OU ++ mktemp + local LAST_ERR=/tmp/tmp.KHJw0dQJfs + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.bx4Lfln2OU serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.KHJw0dQJfs + rm /tmp/tmp.bx4Lfln2OU /tmp/tmp.KHJw0dQJfs + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-5454 + retry=0 + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.hrHvkcoRzD ++ mktemp + local LAST_ERR=/tmp/tmp.nA66LGrrXa + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hrHvkcoRzD serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.nA66LGrrXa + rm /tmp/tmp.hrHvkcoRzD /tmp/tmp.nA66LGrrXa + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.SBwtulN8In +++ mktemp ++ local LAST_ERR=/tmp/tmp.E83crTzhf7 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.SBwtulN8In ++ cat /tmp/tmp.E83crTzhf7 ++ rm /tmp/tmp.SBwtulN8In /tmp/tmp.E83crTzhf7 ++ return 0 + wait_pod postgres-operator-689d766496-4dl4l + local pod=postgres-operator-689d766496-4dl4l + local ns= + set +o xtrace postgres-operator-689d766496-4dl4ltrue + [[ -n '' ]] ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.JNWO4t8dcO +++ mktemp ++ local LAST_ERR=/tmp/tmp.jdSRNf60tq ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.JNWO4t8dcO ++ cat /tmp/tmp.jdSRNf60tq ++ rm /tmp/tmp.JNWO4t8dcO /tmp/tmp.jdSRNf60tq ++ return 0 + [[ xsome-name == \x ]] + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"}, {"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"}, {"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"}, {"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"}, {"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"}, {"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.75VlMwPLQO ++ mktemp + local LAST_ERR=/tmp/tmp.tzxWvQVSCe + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo"}, {"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest"}, {"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger"}, {"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer"}, {"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha"}, {"op":"replace","path":"/metadata/labels/pgo-version","value":"1.5.1"} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.75VlMwPLQO perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.tzxWvQVSCe + rm /tmp/tmp.75VlMwPLQO /tmp/tmp.tzxWvQVSCe + return 0 + [[ xupdate == \x\u\p\d\a\t\e ]] + compare_generation 2 deployment some-name-pgbouncer + local generation=2 + local resource=deployment + local name=some-name-pgbouncer + local current_generation ++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.yjuaQpfeAm +++ mktemp ++ local LAST_ERR=/tmp/tmp.8AnGnaJiCh ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.yjuaQpfeAm ++ cat /tmp/tmp.8AnGnaJiCh ++ rm /tmp/tmp.yjuaQpfeAm /tmp/tmp.8AnGnaJiCh ++ return 0 + current_generation=1 + set +o xtrace .....................................................................................................+ wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HSSeqRTGrI +++ mktemp ++ local LAST_ERR=/tmp/tmp.JYaf2gF4eK ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.HSSeqRTGrI ++ cat /tmp/tmp.JYaf2gF4eK ++ rm /tmp/tmp.HSSeqRTGrI /tmp/tmp.JYaf2gF4eK ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.21nZn4B5c0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ET6uTevpSg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.21nZn4B5c0 +++ cat /tmp/tmp.ET6uTevpSg +++ rm /tmp/tmp.21nZn4B5c0 /tmp/tmp.ET6uTevpSg +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name + sleep 10 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.Ii44ov7bF2 ++ mktemp + local LAST_ERR=/tmp/tmp.PuxdBNz76R + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Ii44ov7bF2 perconapgcluster.pg.percona.com/some-name patched (no change) + cat /tmp/tmp.PuxdBNz76R + rm /tmp/tmp.Ii44ov7bF2 /tmp/tmp.PuxdBNz76R + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.g93FuxiVQL +++ mktemp ++ local LAST_ERR=/tmp/tmp.X4zNn0vpVX ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.g93FuxiVQL ++ cat /tmp/tmp.X4zNn0vpVX ++ rm /tmp/tmp.g93FuxiVQL /tmp/tmp.X4zNn0vpVX ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j8KxV7ihIa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UqFLacw7nL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.j8KxV7ihIa +++ cat /tmp/tmp.UqFLacw7nL +++ rm /tmp/tmp.j8KxV7ihIa /tmp/tmp.UqFLacw7nL +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name-pgbouncer + LATEST_TAG=1.6.0 + IMAGE_OPERATOR=perconalab/percona-postgresql-operator:PR-757-14c649e4c-postgres-operator + IMAGE_DEPLOYER=perconalab/percona-postgresql-operator:PR-757-14c649e4c-pgo-deployer + IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + IMAGE_PG_HA=perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + IMAGE_BACKREST_REPO=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + IMAGE_PGBADGER=perconalab/percona-postgresql-operator:main-ppg14-pgbadger + desc 'update operator' + set +o xtrace ----------------------------------------------------------------------------------- update operator ----------------------------------------------------------------------------------- + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml disabled update + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml + local namespace_mode=disabled + local action=update + local namespace=upgrade-5454 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml disabled update upgrade-5454 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=update + local namespace=upgrade-5454 + local dont_send_telemetry=false + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-5454"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + cat /dev/fd/63 - ++ cut -d: -f2 ++ echo perconalab/percona-postgresql-operator:PR-757-14c649e4c + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-5454"#g' ++ echo 'crunchy_debug: "true"' + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-757-14c649e4c"#g' ++ echo perconalab/percona-postgresql-operator:PR-757-14c649e4c ++ cut -d: -f1 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-757/deploy/operator.yaml metadata.namespace upgrade-5454 + yq w -d3 - 'subjects[0].namespace' upgrade-5454 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-757-14c649e4c-pgo-deployer ++ cat /tmp/tmp.RFCZbnUcQr/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-5454" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-757-14c649e4c" pgo_operator_namespace: "upgrade-5454"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Yp2xQ0B4j8 ++ mktemp + local LAST_ERR=/tmp/tmp.vg5E7VZSoR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl apply -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Yp2xQ0B4j8 serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.vg5E7VZSoR + rm /tmp/tmp.Yp2xQ0B4j8 /tmp/tmp.vg5E7VZSoR + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-5454 + retry=0 + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 53 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 54 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 55 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 56 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 57 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 58 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 59 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 60 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 61 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 62 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 63 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 64 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 65 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 66 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 67 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 68 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 69 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 70 -ge 600 ']' ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-5454 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.JvlspoBbTb ++ mktemp + local LAST_ERR=/tmp/tmp.tyBvzwWmat + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl delete -f /tmp/tmp.RFCZbnUcQr/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.JvlspoBbTb serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.tyBvzwWmat + rm /tmp/tmp.JvlspoBbTb /tmp/tmp.tyBvzwWmat + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vEj59NdojF +++ mktemp ++ local LAST_ERR=/tmp/tmp.EbLzJSZ9sy ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.vEj59NdojF ++ cat /tmp/tmp.EbLzJSZ9sy ++ rm /tmp/tmp.vEj59NdojF /tmp/tmp.EbLzJSZ9sy ++ return 0 + wait_pod postgres-operator-5c6b8f65d4-r95t7 + local pod=postgres-operator-5c6b8f65d4-r95t7 + local ns= + set +o xtrace postgres-operator-5c6b8f65d4-r95t7true + [[ -n '' ]] + kubectl_bin diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.O8RbhTEy3O ++ mktemp + local LAST_ERR=/tmp/tmp.H5LYwhWeOY + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.O8RbhTEy3O + cat /tmp/tmp.H5LYwhWeOY + rm /tmp/tmp.O8RbhTEy3O /tmp/tmp.H5LYwhWeOY + return 0 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo}, {"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest}, {"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger}, {"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer}, {"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha}, {"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.HMMjzttRw2 ++ mktemp + local LAST_ERR=/tmp/tmp.JLJtM5rsqQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo}, {"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest}, {"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger}, {"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer}, {"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha}, {"op":"replace","path":"/metadata/labels/pgo-version","value":1.6.0} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HMMjzttRw2 perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.JLJtM5rsqQ + rm /tmp/tmp.HMMjzttRw2 /tmp/tmp.JLJtM5rsqQ + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-5454 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.HtgZLvBkES +++ mktemp ++ local LAST_ERR=/tmp/tmp.BCzD2pojUv ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.HtgZLvBkES ++ cat /tmp/tmp.BCzD2pojUv ++ rm /tmp/tmp.HtgZLvBkES /tmp/tmp.BCzD2pojUv ++ return 0 + status_query='kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Nft68pq7L1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KsZ6hw5iOV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade +++ kubectl -n upgrade-5454 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Nft68pq7L1 +++ cat /tmp/tmp.KsZ6hw5iOV +++ rm /tmp/tmp.Nft68pq7L1 /tmp/tmp.KsZ6hw5iOV +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-5454 + sleep 10 + set +o xtrace some-name + compare_generation 3 deployment some-name-pgbouncer + local generation=3 + local resource=deployment + local name=some-name-pgbouncer + local current_generation ++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hlkpIJOp2Z +++ mktemp ++ local LAST_ERR=/tmp/tmp.tjTmhaUW4y ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-757-14c649e4c-3-upgrade ++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.hlkpIJOp2Z ++ cat /tmp/tmp.tjTmhaUW4y ++ rm /tmp/tmp.hlkpIJOp2Z /tmp/tmp.tjTmhaUW4y ++ return 0 + current_generation=2 + set +o xtrace ........................................................................................................................................................................................................................................................................................................................................................................max retry count 360 reached. something went wrong with operator or kubernetes cluster Generation for resource type deployment with name some-name-pgbouncer is: 2, but should be: 3!