++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/upgrade.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/upgrade.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.15.1 ++ '[' v3 == v2 ']' + main + create_namespace upgrade-21023 + local namespace=upgrade-21023 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + cleanup_rbac + awk '{print$1}' + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.vshwlXhsq3 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.ewReu0xISc + local LAST_ERR=/tmp/tmp.3WS20RjfWY + local exit_status=0 + local timeout=4 + set +o errexit ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.5H3XvC6rww + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl get ns + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.vshwlXhsq3 + cat /tmp/tmp.3WS20RjfWY + rm /tmp/tmp.vshwlXhsq3 /tmp/tmp.3WS20RjfWY + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level namespace "upgrade-8151" deleted + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.ewReu0xISc + cat /tmp/tmp.5H3XvC6rww Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.ewReu0xISc /tmp/tmp.5H3XvC6rww + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.PFwZRyZWla ++ mktemp + local LAST_ERR=/tmp/tmp.OHuEXB8mzj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.PFwZRyZWla + cat /tmp/tmp.OHuEXB8mzj Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.PFwZRyZWla /tmp/tmp.OHuEXB8mzj + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace upgrade-21023 ++ mktemp + local LAST_OUT=/tmp/tmp.xxU2uba4L3 ++ mktemp + local LAST_ERR=/tmp/tmp.wlqpVxMhAg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete namespace upgrade-21023 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete namespace upgrade-21023 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete namespace upgrade-21023 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.xxU2uba4L3 + cat /tmp/tmp.wlqpVxMhAg Error from server (NotFound): namespaces "upgrade-21023" not found + rm /tmp/tmp.xxU2uba4L3 /tmp/tmp.wlqpVxMhAg + return 1 + : + wait_for_delete namespace/upgrade-21023 + local res=namespace/upgrade-21023 + set +o xtrace namespace/upgrade-21023 - Error from server (NotFound): namespaces "upgrade-21023" not found + kubectl_bin create namespace upgrade-21023 ++ mktemp + local LAST_OUT=/tmp/tmp.ulXfroMd7w ++ mktemp + local LAST_ERR=/tmp/tmp.X1bGNp9N3R + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl create namespace upgrade-21023 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ulXfroMd7w namespace/upgrade-21023 created + cat /tmp/tmp.X1bGNp9N3R + rm /tmp/tmp.ulXfroMd7w /tmp/tmp.X1bGNp9N3R + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.e2tlhanOlV +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wpn2he75Bw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.e2tlhanOlV ++ cat /tmp/tmp.Wpn2he75Bw ++ rm /tmp/tmp.e2tlhanOlV /tmp/tmp.Wpn2he75Bw ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-upgrade --namespace=upgrade-21023 ++ mktemp + local LAST_OUT=/tmp/tmp.7Z5WQrWRx5 ++ mktemp + local LAST_ERR=/tmp/tmp.sJPrVaiw7y + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-upgrade --namespace=upgrade-21023 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7Z5WQrWRx5 Context "gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-upgrade" modified. + cat /tmp/tmp.sJPrVaiw7y + rm /tmp/tmp.7Z5WQrWRx5 /tmp/tmp.sJPrVaiw7y + return 0 + cluster=some-name + operator_action=install + create_or_upgrade_to_version some-name 1.5.1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml dynamic install + local cluster=some-name + local pgo_tag=1.5.1 + local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml + local namespace_mode=dynamic + local operator_action=install + IMAGE_OPERATOR=percona/percona-postgresql-operator:1.5.1-postgres-operator + IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.5.1-pgo-deployer + IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer + IMAGE_PG_HA=percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha + IMAGE_BACKREST=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest + IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo + IMAGE_PGBADGER=percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger + [[ xinstall == \x\i\n\s\t\a\l\l ]] + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml dynamic + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml + local namespace_mode=dynamic + local action=install + local namespace=upgrade-21023 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml dynamic install upgrade-21023 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml + local namespace_mode=dynamic + local operator_action=install + local namespace=upgrade-21023 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml 'data[values.yaml]' ++ cut -d: -f2 ++ echo percona/percona-postgresql-operator:1.5.1 + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.5.1"#g' + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-21023"#g' ++ cut -d: -f1 + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-21023"#g' ++ echo percona/percona-postgresql-operator:1.5.1 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g' + cat /dev/fd/63 - ++ echo 'crunchy_debug: "true"' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w -d3 - 'subjects[0].namespace' upgrade-21023 ++ cat /tmp/tmp.V8NoLLRtUg/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-21023" namespace_mode: "dynamic" pgo_image_prefix: "percona/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "1.5.1" pgo_operator_namespace: "upgrade-21023"' + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_operator.yaml metadata.namespace upgrade-21023 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install + yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.5.1-pgo-deployer + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.BtoFrptre3 ++ mktemp + local LAST_ERR=/tmp/tmp.NUr4IGYXwk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BtoFrptre3 serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.NUr4IGYXwk + rm /tmp/tmp.BtoFrptre3 /tmp/tmp.NUr4IGYXwk + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.xtI98HgOkA ++ mktemp + local LAST_ERR=/tmp/tmp.FMVQsEqO00 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xtI98HgOkA serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.FMVQsEqO00 + rm /tmp/tmp.xtI98HgOkA /tmp/tmp.FMVQsEqO00 + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.jiTdT1A8kF +++ mktemp ++ local LAST_ERR=/tmp/tmp.btX61x2p5V ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.jiTdT1A8kF ++ cat /tmp/tmp.btX61x2p5V ++ rm /tmp/tmp.jiTdT1A8kF /tmp/tmp.btX61x2p5V ++ return 0 + wait_pod postgres-operator-5576576bb4-k5qjb + local pod=postgres-operator-5576576bb4-k5qjb + local ns= + set +o xtrace postgres-operator-5576576bb4-k5qjbtrue + [[ -n '' ]] ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mg1A7fnOOd +++ mktemp ++ local LAST_ERR=/tmp/tmp.73i3R9actT ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 0 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 4 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=1 ++ [[ 1 != 0 ]] ++ sleep 8 ++ set -o errexit ++ cat /tmp/tmp.mg1A7fnOOd ++ cat /tmp/tmp.73i3R9actT Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found ++ rm /tmp/tmp.mg1A7fnOOd /tmp/tmp.73i3R9actT ++ return 1 + [[ x == \x ]] + desc 'start cluster' + set +o xtrace ----------------------------------------------------------------------------------- start cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name + local cluster=some-name + local namespace=upgrade-21023 + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n upgrade-21023 apply -f - + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.SngZcXy99p ++ mktemp + local LAST_ERR=/tmp/tmp.yYGD0px0Yw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.SngZcXy99p secret/some-name-users-secret created + cat /tmp/tmp.yYGD0px0Yw + rm /tmp/tmp.SngZcXy99p /tmp/tmp.yYGD0px0Yw + return 0 + yq w - metadata.name some-name-pmm-secret + kubectl_bin -n upgrade-21023 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.7JOjMXG4Nu ++ mktemp + local LAST_ERR=/tmp/tmp.aJ9Cv3hbNj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 apply -f - + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7JOjMXG4Nu secret/some-name-pmm-secret created + cat /tmp/tmp.aJ9Cv3hbNj + rm /tmp/tmp.7JOjMXG4Nu /tmp/tmp.aJ9Cv3hbNj + return 0 + spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_cr.yaml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_cr.yaml + local backup=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=upgrade-21023 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name upgrade-21023 + local cluster=some-name + local namespace=upgrade-21023 + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n upgrade-21023 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.BHg0CR3M3Q ++ mktemp + local LAST_ERR=/tmp/tmp.kSrm7YAEGc + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BHg0CR3M3Q secret/some-name-users-secret configured + cat /tmp/tmp.kSrm7YAEGc + rm /tmp/tmp.BHg0CR3M3Q /tmp/tmp.kSrm7YAEGc + return 0 + yq w - metadata.name some-name-pmm-secret + kubectl_bin -n upgrade-21023 apply -f - + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.PIJ9uvLUq6 ++ mktemp + local LAST_ERR=/tmp/tmp.aFoH2AwagV + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.PIJ9uvLUq6 secret/some-name-pmm-secret configured + cat /tmp/tmp.aFoH2AwagV + rm /tmp/tmp.PIJ9uvLUq6 /tmp/tmp.aFoH2AwagV + return 0 + kubectl_bin -n upgrade-21023 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8KQugzaiww ++ mktemp + local LAST_ERR=/tmp/tmp.DeZj5Zbuym + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8KQugzaiww deployment.apps/pg-client created + cat /tmp/tmp.DeZj5Zbuym + rm /tmp/tmp.8KQugzaiww /tmp/tmp.DeZj5Zbuym + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_cr.yaml some-name false false false false false false false false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local root_via_pgbouncer= + yq w - spec.pgBadger.image percona/percona-postgresql-operator:1.5.1-ppg14-pgbadger + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - spec.pgBouncer.image percona/percona-postgresql-operator:1.5.1-ppg14-pgbouncer + yq w - spec.backup.backrestRepoImage percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest-repo + yq w - spec.name some-name + yq w - spec.database some-name + yq w - spec.keepData false + yq w - metadata.labels.name some-name + yq w - spec.clustername some-name + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.keepBackups false + yq w - spec.user some-name + yq w - spec.secretsName some-name-users-secret + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.5.1_cr.yaml metadata.annotations.current-primary some-name + yq w - spec.pgPrimary.image percona/percona-postgresql-operator:1.5.1-ppg14-postgres-ha + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - spec.backup.image percona/percona-postgresql-operator:1.5.1-ppg14-pgbackrest + yq w - metadata.labels.deployment-name some-name + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.V8NoLLRtUg/cr.yaml -n upgrade-21023 ++ mktemp + local LAST_OUT=/tmp/tmp.yjegSLibnK ++ mktemp + local LAST_ERR=/tmp/tmp.tsKaJoC3hk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /tmp/tmp.V8NoLLRtUg/cr.yaml -n upgrade-21023 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yjegSLibnK perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.tsKaJoC3hk + rm /tmp/tmp.yjegSLibnK /tmp/tmp.tsKaJoC3hk + return 0 + wait_deployment some-name-backrest-shared-repo + local name=some-name-backrest-shared-repo + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name-backrest-shared-repo. + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name...... + [[ false == \g\c\s\+* ]] + wait_job_completion some-name-stanza-create false + local job=some-name-stanza-create + local mandatory=false + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' some-name-stanza-create+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name + local job=backrest-backup-some-name + local mandatory=true + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' backrest-backup-some-name+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Mhz4HKPLBP +++ mktemp ++ local LAST_ERR=/tmp/tmp.jDoxyQkr31 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Mhz4HKPLBP ++ cat /tmp/tmp.jDoxyQkr31 ++ rm /tmp/tmp.Mhz4HKPLBP /tmp/tmp.jDoxyQkr31 ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ucSCY8zI0f ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HgL0YFotXo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ucSCY8zI0f +++ cat /tmp/tmp.HgL0YFotXo +++ rm /tmp/tmp.ucSCY8zI0f /tmp/tmp.HgL0YFotXo +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GjESAAljMZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SHP80gkz8R +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.GjESAAljMZ +++ cat /tmp/tmp.SHP80gkz8R +++ rm /tmp/tmp.GjESAAljMZ /tmp/tmp.SHP80gkz8R +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-21023 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nSPSc9UUWm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VX65ymB6tA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.nSPSc9UUWm +++ cat /tmp/tmp.VX65ymB6tA +++ rm /tmp/tmp.nSPSc9UUWm /tmp/tmp.VX65ymB6tA +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GoSYUyn5O2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1tmC0jvyJf +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.GoSYUyn5O2 +++ cat /tmp/tmp.1tmC0jvyJf +++ rm /tmp/tmp.GoSYUyn5O2 /tmp/tmp.1tmC0jvyJf +++ return 0 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fWUUfmJyNU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FzrU8DZTi0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fWUUfmJyNU +++ cat /tmp/tmp.FzrU8DZTi0 +++ rm /tmp/tmp.fWUUfmJyNU /tmp/tmp.FzrU8DZTi0 +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.JfMzs5z9R0 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.hft53fPR6j ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.JfMzs5z9R0 ++++ cat /tmp/tmp.hft53fPR6j ++++ rm /tmp/tmp.JfMzs5z9R0 /tmp/tmp.hft53fPR6j ++++ return 0 +++ kubectl_bin -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bwXyR5Xt1t ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VvQY28739x +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bwXyR5Xt1t +++ cat /tmp/tmp.VvQY28739x +++ rm /tmp/tmp.bwXyR5Xt1t /tmp/tmp.VvQY28739x +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.upgrade-21023 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name.upgrade-21023 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.XMzZ2R97tt +++ mktemp ++ local LAST_ERR=/tmp/tmp.TMGYb88Yd3 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.XMzZ2R97tt ++ cat /tmp/tmp.TMGYb88Yd3 ++ rm /tmp/tmp.XMzZ2R97tt /tmp/tmp.TMGYb88Yd3 ++ return 0 + local client_container=pg-client-76df9df876-fp4nb + kubectl_bin exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.jNRGrPsNib ++ mktemp + local LAST_ERR=/tmp/tmp.iEaKSdecgA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jNRGrPsNib + cat /tmp/tmp.iEaKSdecgA + rm /tmp/tmp.jNRGrPsNib /tmp/tmp.iEaKSdecgA + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-21023 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SfUJQgFLxX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2sIlCrbvH3 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.SfUJQgFLxX +++ cat /tmp/tmp.2sIlCrbvH3 +++ rm /tmp/tmp.SfUJQgFLxX /tmp/tmp.2sIlCrbvH3 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Pb5KFK17YU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DEDkqiWX8m +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Pb5KFK17YU +++ cat /tmp/tmp.DEDkqiWX8m +++ rm /tmp/tmp.Pb5KFK17YU /tmp/tmp.DEDkqiWX8m +++ return 0 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LAr0nwDdZr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7FtZ9BbVfN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.LAr0nwDdZr +++ cat /tmp/tmp.7FtZ9BbVfN +++ rm /tmp/tmp.LAr0nwDdZr /tmp/tmp.7FtZ9BbVfN +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.S9OlM5qBoE +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PGFp2ox1Te ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.S9OlM5qBoE ++++ cat /tmp/tmp.PGFp2ox1Te ++++ rm /tmp/tmp.S9OlM5qBoE /tmp/tmp.PGFp2ox1Te ++++ return 0 +++ kubectl_bin -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PX7GRHOTsK ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EAVeFjf8OD +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PX7GRHOTsK +++ cat /tmp/tmp.EAVeFjf8OD +++ rm /tmp/tmp.PX7GRHOTsK /tmp/tmp.EAVeFjf8OD +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.upgrade-21023 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name.upgrade-21023 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eBw1cdmTUx +++ mktemp ++ local LAST_ERR=/tmp/tmp.xmnyYh2K9w ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.eBw1cdmTUx ++ cat /tmp/tmp.xmnyYh2K9w ++ rm /tmp/tmp.eBw1cdmTUx /tmp/tmp.xmnyYh2K9w ++ return 0 + local client_container=pg-client-76df9df876-fp4nb + kubectl_bin exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.cWDayBks0c ++ mktemp + local LAST_ERR=/tmp/tmp.5IdPkrbkRx + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.cWDayBks0c + cat /tmp/tmp.5IdPkrbkRx + rm /tmp/tmp.cWDayBks0c /tmp/tmp.5IdPkrbkRx + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=upgrade-21023 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tIVNsTZqJD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Dks5hLMFgn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tIVNsTZqJD +++ cat /tmp/tmp.Dks5hLMFgn +++ rm /tmp/tmp.tIVNsTZqJD /tmp/tmp.Dks5hLMFgn +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ekxCmvyIk2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0pNpNuUFOl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ekxCmvyIk2 +++ cat /tmp/tmp.0pNpNuUFOl +++ rm /tmp/tmp.ekxCmvyIk2 /tmp/tmp.0pNpNuUFOl +++ return 0 +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bky4bnXt4r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.J7dPg9g28c +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bky4bnXt4r +++ cat /tmp/tmp.J7dPg9g28c +++ rm /tmp/tmp.bky4bnXt4r /tmp/tmp.J7dPg9g28c +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ehg2LwQwaA +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8vNyhqoR02 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ehg2LwQwaA ++++ cat /tmp/tmp.8vNyhqoR02 ++++ rm /tmp/tmp.ehg2LwQwaA /tmp/tmp.8vNyhqoR02 ++++ return 0 +++ kubectl_bin -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zgvH9VLTRo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.l22AAFszII +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zgvH9VLTRo +++ cat /tmp/tmp.l22AAFszII +++ rm /tmp/tmp.zgvH9VLTRo /tmp/tmp.l22AAFszII +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.upgrade-21023 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name.upgrade-21023 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1f0Ho2hJ1O +++ mktemp ++ local LAST_ERR=/tmp/tmp.ABygcZW9Bi ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.1f0Ho2hJ1O ++ cat /tmp/tmp.ABygcZW9Bi ++ rm /tmp/tmp.1f0Ho2hJ1O /tmp/tmp.ABygcZW9Bi ++ return 0 + local client_container=pg-client-76df9df876-fp4nb + kubectl_bin exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.HfulpTAWID ++ mktemp + local LAST_ERR=/tmp/tmp.n0DL3RTrZW + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.upgrade-21023.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HfulpTAWID + cat /tmp/tmp.n0DL3RTrZW + rm /tmp/tmp.HfulpTAWID /tmp/tmp.n0DL3RTrZW + return 0 + sleep 10 + [[ xinstall == \x\u\p\d\a\t\e ]] + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.V1VRukT25k +++ mktemp ++ local LAST_ERR=/tmp/tmp.DVl30Wiw1z ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.V1VRukT25k ++ cat /tmp/tmp.DVl30Wiw1z ++ rm /tmp/tmp.V1VRukT25k /tmp/tmp.DVl30Wiw1z ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Pd2sA1QtM6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bZ0pZw1BsG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Pd2sA1QtM6 +++ cat /tmp/tmp.bZ0pZw1BsG +++ rm /tmp/tmp.Pd2sA1QtM6 /tmp/tmp.bZ0pZw1BsG +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name + sleep 10 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.SI8ZeJ0g8z ++ mktemp + local LAST_ERR=/tmp/tmp.3KrXwFKUGp + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.SI8ZeJ0g8z perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.3KrXwFKUGp + rm /tmp/tmp.SI8ZeJ0g8z /tmp/tmp.3KrXwFKUGp + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.DUpKVJGd5v +++ mktemp ++ local LAST_ERR=/tmp/tmp.595myaG9mS ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.DUpKVJGd5v ++ cat /tmp/tmp.595myaG9mS ++ rm /tmp/tmp.DUpKVJGd5v /tmp/tmp.595myaG9mS ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.1pABS86azh ++++ mktemp +++ local LAST_ERR=/tmp/tmp.JgcTjT7RFc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.1pABS86azh +++ cat /tmp/tmp.JgcTjT7RFc +++ rm /tmp/tmp.1pABS86azh /tmp/tmp.JgcTjT7RFc +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name-pgbouncer.. + operator_action=update + create_or_upgrade_to_version some-name 1.6.0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml dynamic update + local cluster=some-name + local pgo_tag=1.6.0 + local operator_yaml=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml + local namespace_mode=dynamic + local operator_action=update + IMAGE_OPERATOR=percona/percona-postgresql-operator:1.6.0-postgres-operator + IMAGE_DEPLOYER=percona/percona-postgresql-operator:1.6.0-pgo-deployer + IMAGE_PGBOUNCER=percona/percona-postgresql-operator:1.6.0-ppg14-pgbouncer + IMAGE_PG_HA=percona/percona-postgresql-operator:1.6.0-ppg14-postgres-ha + IMAGE_BACKREST=percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest + IMAGE_BACKREST_REPO=percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest-repo + IMAGE_PGBADGER=percona/percona-postgresql-operator:1.6.0-ppg14-pgbadger + [[ xupdate == \x\i\n\s\t\a\l\l ]] + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml dynamic update + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml + local namespace_mode=dynamic + local action=update + local namespace=upgrade-21023 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml dynamic update upgrade-21023 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml + local namespace_mode=dynamic + local operator_action=update + local namespace=upgrade-21023 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-21023"#g' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "dynamic"#g' ++ cut -d: -f2 ++ echo percona/percona-postgresql-operator:1.6.0 + cat /dev/fd/63 - + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "1.6.0"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-21023"#g' ++ echo 'crunchy_debug: "true"' ++ cut -d: -f1 ++ echo percona/percona-postgresql-operator:1.6.0 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "percona/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/1.6.0_operator.yaml metadata.namespace upgrade-21023 + yq w -d4 - 'spec.template.spec.containers[0].image' percona/percona-postgresql-operator:1.6.0-pgo-deployer + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update + yq w -d3 - 'subjects[0].namespace' upgrade-21023 ++ cat /tmp/tmp.V8NoLLRtUg/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-21023" namespace_mode: "dynamic" pgo_image_prefix: "percona/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "1.6.0" pgo_operator_namespace: "upgrade-21023"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.C05ny3jsAx ++ mktemp + local LAST_ERR=/tmp/tmp.HLihXjgWzq + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.C05ny3jsAx serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.HLihXjgWzq + rm /tmp/tmp.C05ny3jsAx /tmp/tmp.HLihXjgWzq + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.gdL4ZkPLLf ++ mktemp + local LAST_ERR=/tmp/tmp.Dgzp93KIZy + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.gdL4ZkPLLf serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.Dgzp93KIZy + rm /tmp/tmp.gdL4ZkPLLf /tmp/tmp.Dgzp93KIZy + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xg6YpyrcMp +++ mktemp ++ local LAST_ERR=/tmp/tmp.qtoNe2OTHX ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xg6YpyrcMp ++ cat /tmp/tmp.qtoNe2OTHX ++ rm /tmp/tmp.xg6YpyrcMp /tmp/tmp.qtoNe2OTHX ++ return 0 + wait_pod postgres-operator-9595457f6-jjrkp + local pod=postgres-operator-9595457f6-jjrkp + local ns= + set +o xtrace postgres-operator-9595457f6-jjrkptrue + [[ -n '' ]] ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.KtTlMC5qq0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.cvRjUPNilB ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.KtTlMC5qq0 ++ cat /tmp/tmp.cvRjUPNilB ++ rm /tmp/tmp.KtTlMC5qq0 /tmp/tmp.cvRjUPNilB ++ return 0 + [[ xsome-name == \x ]] + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest-repo"}, {"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest"}, {"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbadger"}, {"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbouncer"}, {"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-postgres-ha"}, {"op":"replace","path":"/metadata/labels/pgo-version","value":"1.6.0"} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.X0VW2mb1t9 ++ mktemp + local LAST_ERR=/tmp/tmp.xPabdtP5Vf + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest-repo"}, {"op":"replace","path":"/spec/backup/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbackrest"}, {"op":"replace","path":"/spec/pgBadger/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbadger"}, {"op":"replace","path":"/spec/pgBouncer/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-pgbouncer"}, {"op":"replace","path":"/spec/pgPrimary/image","value":"percona/percona-postgresql-operator:1.6.0-ppg14-postgres-ha"}, {"op":"replace","path":"/metadata/labels/pgo-version","value":"1.6.0"} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.X0VW2mb1t9 perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.xPabdtP5Vf + rm /tmp/tmp.X0VW2mb1t9 /tmp/tmp.xPabdtP5Vf + return 0 + [[ xupdate == \x\u\p\d\a\t\e ]] + compare_generation 2 deployment some-name-pgbouncer + local generation=2 + local resource=deployment + local name=some-name-pgbouncer + local current_generation ++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.iKGxZFP8zT +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZrTfnRsQxC ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.iKGxZFP8zT ++ cat /tmp/tmp.ZrTfnRsQxC ++ rm /tmp/tmp.iKGxZFP8zT /tmp/tmp.ZrTfnRsQxC ++ return 0 + current_generation=1 + set +o xtrace ...........................................................................................+ wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ju3e4Xv6SU +++ mktemp ++ local LAST_ERR=/tmp/tmp.7HpguImruV ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Ju3e4Xv6SU ++ cat /tmp/tmp.7HpguImruV ++ rm /tmp/tmp.Ju3e4Xv6SU /tmp/tmp.7HpguImruV ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yTVszptq4q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IwKc1WcqjU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.yTVszptq4q +++ cat /tmp/tmp.IwKc1WcqjU +++ rm /tmp/tmp.yTVszptq4q /tmp/tmp.IwKc1WcqjU +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name + sleep 10 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.LsbueLb39i ++ mktemp + local LAST_ERR=/tmp/tmp.heedSHrkOe + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/pgBouncer/size","value":1} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.LsbueLb39i perconapgcluster.pg.percona.com/some-name patched (no change) + cat /tmp/tmp.heedSHrkOe + rm /tmp/tmp.LsbueLb39i /tmp/tmp.heedSHrkOe + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mVFZ2rdtYQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.Mo59xpJUBO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.mVFZ2rdtYQ ++ cat /tmp/tmp.Mo59xpJUBO ++ rm /tmp/tmp.mVFZ2rdtYQ /tmp/tmp.Mo59xpJUBO ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TtzwB0tzDX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qIJG5xbgfP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TtzwB0tzDX +++ cat /tmp/tmp.qIJG5xbgfP +++ rm /tmp/tmp.TtzwB0tzDX /tmp/tmp.qIJG5xbgfP +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name-pgbouncer + LATEST_TAG=1.7.0 + IMAGE_OPERATOR=perconalab/percona-postgresql-operator:PR-776-0087ef129-postgres-operator + IMAGE_DEPLOYER=perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-deployer + IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + IMAGE_PG_HA=perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + IMAGE_BACKREST_REPO=perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + IMAGE_PGBADGER=perconalab/percona-postgresql-operator:main-ppg14-pgbadger + desc 'update operator' + set +o xtrace ----------------------------------------------------------------------------------- update operator ----------------------------------------------------------------------------------- + deploy_operator '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml disabled update + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local action=update + local namespace=upgrade-21023 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml disabled update upgrade-21023 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=update + local namespace=upgrade-21023 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^namespace: .*#namespace: "upgrade-21023"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + cat /dev/fd/63 - ++ cut -d: -f2 + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "upgrade-21023"#g' ++ echo 'crunchy_debug: "true"' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml 'data[values.yaml]' ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-776-0087ef129"#g' ++ cut -d: -f1 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml metadata.namespace upgrade-21023 + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-deployer + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' update + yq w -d3 - 'subjects[0].namespace' upgrade-21023 ++ cat /tmp/tmp.V8NoLLRtUg/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "upgrade-21023" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-776-0087ef129" pgo_operator_namespace: "upgrade-21023"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.c1fFzOtHr2 ++ mktemp + local LAST_ERR=/tmp/tmp.mOmkyXAGKj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.c1fFzOtHr2 serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.mOmkyXAGKj + rm /tmp/tmp.c1fFzOtHr2 /tmp/tmp.mOmkyXAGKj + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.RB1AV4O8t3 ++ mktemp + local LAST_ERR=/tmp/tmp.hHlkAQjiDK + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.RB1AV4O8t3 serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.hHlkAQjiDK + rm /tmp/tmp.RB1AV4O8t3 /tmp/tmp.hHlkAQjiDK + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ntq4a4HDPZ +++ mktemp ++ local LAST_ERR=/tmp/tmp.UWKJ6U6x3P ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ntq4a4HDPZ ++ cat /tmp/tmp.UWKJ6U6x3P ++ rm /tmp/tmp.ntq4a4HDPZ /tmp/tmp.UWKJ6U6x3P ++ return 0 + wait_pod postgres-operator-5d57849c8d-9qx2r + local pod=postgres-operator-5d57849c8d-9qx2r + local ns= + set +o xtrace postgres-operator-5d57849c8d-9qx2rtrue + [[ -n '' ]] + kubectl_bin diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.UJYdh3AnOb ++ mktemp + local LAST_ERR=/tmp/tmp.tzNNLOMUzk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl diff -f installers/ansible/roles/pgo-operator/files/crds/perconapgclusters-crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.UJYdh3AnOb + cat /tmp/tmp.tzNNLOMUzk + rm /tmp/tmp.UJYdh3AnOb /tmp/tmp.tzNNLOMUzk + return 0 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo}, {"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest}, {"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger}, {"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer}, {"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha}, {"op":"replace","path":"/metadata/labels/pgo-version","value":1.7.0} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.4LPJmG6dJ3 ++ mktemp + local LAST_ERR=/tmp/tmp.upboKpoKLG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"replace","path":"/spec/backup/backrestRepoImage","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo}, {"op":"replace","path":"/spec/backup/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbackrest}, {"op":"replace","path":"/spec/pgBadger/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbadger}, {"op":"replace","path":"/spec/pgBouncer/image","value":perconalab/percona-postgresql-operator:main-ppg14-pgbouncer}, {"op":"replace","path":"/spec/pgPrimary/image","value":perconalab/percona-postgresql-operator:main-ppg14-postgres-ha}, {"op":"replace","path":"/metadata/labels/pgo-version","value":1.7.0} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.4LPJmG6dJ3 perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.upboKpoKLG + rm /tmp/tmp.4LPJmG6dJ3 /tmp/tmp.upboKpoKLG + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.fqoEDaoZne +++ mktemp ++ local LAST_ERR=/tmp/tmp.9o0ASj464F ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.fqoEDaoZne ++ cat /tmp/tmp.9o0ASj464F ++ rm /tmp/tmp.fqoEDaoZne /tmp/tmp.9o0ASj464F ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rhvAvUYupF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.3xeQSYQXuw +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.rhvAvUYupF +++ cat /tmp/tmp.3xeQSYQXuw +++ rm /tmp/tmp.rhvAvUYupF /tmp/tmp.3xeQSYQXuw +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name + compare_generation 3 deployment some-name-pgbouncer + local generation=3 + local resource=deployment + local name=some-name-pgbouncer + local current_generation ++ kubectl_bin get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.nD7sGdg124 +++ mktemp ++ local LAST_ERR=/tmp/tmp.Wbvy92Z08z ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get deployment some-name-pgbouncer -o 'jsonpath={.metadata.generation}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.nD7sGdg124 ++ cat /tmp/tmp.Wbvy92Z08z ++ rm /tmp/tmp.nD7sGdg124 /tmp/tmp.Wbvy92Z08z ++ return 0 + current_generation=2 + set +o xtrace ..........................................................................++ kubectl_bin get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xQvTMJTsbv +++ mktemp ++ local LAST_ERR=/tmp/tmp.BB8iPE5JU3 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get deployment/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xQvTMJTsbv ++ cat /tmp/tmp.BB8iPE5JU3 ++ rm /tmp/tmp.xQvTMJTsbv /tmp/tmp.BB8iPE5JU3 ++ return 0 + [[ 1.7.0 != 1.7.0 ]] ++ kubectl_bin get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wBrQx4P2lb +++ mktemp ++ local LAST_ERR=/tmp/tmp.A70IWHTYJb ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get deployment/some-name -o 'jsonpath={.spec.template.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.wBrQx4P2lb ++ cat /tmp/tmp.A70IWHTYJb ++ rm /tmp/tmp.wBrQx4P2lb /tmp/tmp.A70IWHTYJb ++ return 0 + [[ 1.7.0 != 1.7.0 ]] + sleep 10 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.QMdfRhRN4f +++ mktemp ++ local LAST_ERR=/tmp/tmp.DU5dGaomyx ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.QMdfRhRN4f ++ cat /tmp/tmp.DU5dGaomyx ++ rm /tmp/tmp.QMdfRhRN4f /tmp/tmp.DU5dGaomyx ++ return 0 + local client_container=pg-client-76df9df876-fp4nb + kubectl_bin exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.N1Y7f8H75f ++ mktemp + local LAST_ERR=/tmp/tmp.1tmHmiFBAe + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.N1Y7f8H75f + cat /tmp/tmp.1tmHmiFBAe + rm /tmp/tmp.N1Y7f8H75f /tmp/tmp.1tmHmiFBAe + return 0 + '[' '!' -s /tmp/tmp.V8NoLLRtUg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.V8NoLLRtUg/select-1.sql + backup_time_interval_min=5 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]}, {"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}} ]' ++ mktemp + local LAST_OUT=/tmp/tmp.lROgDPoOPj ++ mktemp + local LAST_ERR=/tmp/tmp.BaPQzEuHOB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"add","path":"/spec/backup/schedule","value":[{"name":"5min-backup","schedule":"*/5 * * * *","keep":3,"type":"full","storage":"local"}]}, {"op":"add","path":"/spec/backup/storages","value":{"local":{"type":"local"}}} ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.lROgDPoOPj perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.BaPQzEuHOB + rm /tmp/tmp.lROgDPoOPj /tmp/tmp.BaPQzEuHOB + return 0 + sleep 10 ++ next_backup_after_sec some-name-5min-backup ++ local conf_map_name=some-name-5min-backup ++++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ETjTLukJRv +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Lqe2b7it9a ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ETjTLukJRv ++++ cat /tmp/tmp.Lqe2b7it9a ++++ rm /tmp/tmp.ETjTLukJRv /tmp/tmp.Lqe2b7it9a ++++ return 0 +++ /usr/bin/date +%s --date=2024-05-24T18:10:48Z --utc ++ local configmap_creation_time=1716574248 +++ jq .schedule +++ kubectl_bin get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}' +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.H15DkuJrxB ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KUK8fC6RwI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl get configmap/some-name-5min-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.H15DkuJrxB +++ cat /tmp/tmp.KUK8fC6RwI +++ rm /tmp/tmp.H15DkuJrxB /tmp/tmp.KUK8fC6RwI +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1716574248 300 +++ local start_time_sec=1716574248 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716574548 +++ local next_interval_min_edge=48 +++ local next_backup_at_sec=1716574500 +++ echo 1716574500 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 18:10:56 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 18:10:56 GMT ' --utc +%s +++ echo 1716574256 ++ local result=244 ++ [[ 244 -gt 0 ]] ++ echo 244 + sleep 244 + kubectl_bin wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.9FIc4gASbq ++ mktemp + local LAST_ERR=/tmp/tmp.AyPKZj3dhr + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl wait job/some-name-full-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9FIc4gASbq job.batch/some-name-full-sch-backup condition met + cat /tmp/tmp.AyPKZj3dhr + rm /tmp/tmp.9FIc4gASbq /tmp/tmp.AyPKZj3dhr + return 0 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.AIyCLaplhx +++ mktemp ++ local LAST_ERR=/tmp/tmp.fps7aKmq9C ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl -n upgrade-21023 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.AIyCLaplhx ++ cat /tmp/tmp.fps7aKmq9C ++ rm /tmp/tmp.AIyCLaplhx /tmp/tmp.fps7aKmq9C ++ return 0 + local client_container=pg-client-76df9df876-fp4nb + kubectl_bin exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.yUy1c59vZl ++ mktemp + local LAST_ERR=/tmp/tmp.poW0VB6p7r + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-76df9df876-fp4nb -n upgrade-21023 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yUy1c59vZl + cat /tmp/tmp.poW0VB6p7r + rm /tmp/tmp.yUy1c59vZl /tmp/tmp.poW0VB6p7r + return 0 + '[' '!' -s /tmp/tmp.V8NoLLRtUg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.V8NoLLRtUg/select-1.sql + kubectl create secret generic some-name-ssl-ca --from-file=ca.crt=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/ca.crt secret/some-name-ssl-ca created + kubectl create secret tls some-name-ssl-keypair --cert=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/server.crt --key=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/server.key secret/some-name-ssl-keypair created + kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.8s49V87dqE ++ mktemp + local LAST_ERR=/tmp/tmp.8OGL7VB0Fz + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8s49V87dqE deployment.apps "pg-client" deleted + cat /tmp/tmp.8OGL7VB0Fz + rm /tmp/tmp.8s49V87dqE /tmp/tmp.8OGL7VB0Fz + return 0 + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.hyg60JrgXp ++ mktemp + local LAST_ERR=/tmp/tmp.iKk0YAcmXo + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.hyg60JrgXp deployment.apps/pg-client created + cat /tmp/tmp.iKk0YAcmXo + rm /tmp/tmp.hyg60JrgXp /tmp/tmp.iKk0YAcmXo + return 0 + kubectl_bin patch perconapgcluster/some-name --type json -p '[ {"op":"add","path":"/spec/tlsOnly","value":true}, {"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''}, {"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''}, {"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''}, {"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''}, ]' ++ mktemp + local LAST_OUT=/tmp/tmp.iKoaKyBraD ++ mktemp + local LAST_ERR=/tmp/tmp.z2CfrO1BFZ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json -p '[ {"op":"add","path":"/spec/tlsOnly","value":true}, {"op":"add","path":"/spec/sslCA","value":'\''some-name-ssl-ca'\''}, {"op":"add","path":"/spec/sslSecretName","value":'\''some-name-ssl-keypair'\''}, {"op":"add","path":"/spec/sslReplicationSecretName","value":'\''some-name-ssl-keypair'\''}, {"op":"add","path":"/spec/pgBouncer/tlsSecret","value":'\''some-name-ssl-keypair'\''}, ]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.iKoaKyBraD perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.z2CfrO1BFZ + rm /tmp/tmp.iKoaKyBraD /tmp/tmp.z2CfrO1BFZ + return 0 + disable_pgBouncer some-name + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.2Kb7EArvT2 ++ mktemp + local LAST_ERR=/tmp/tmp.PL1KDDU6ZI + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2Kb7EArvT2 perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.PL1KDDU6ZI + rm /tmp/tmp.2Kb7EArvT2 /tmp/tmp.PL1KDDU6ZI + return 0 + kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.1c8DubzPGh ++ mktemp + local LAST_ERR=/tmp/tmp.mL0hjyYyCF + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1c8DubzPGh deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.mL0hjyYyCF + rm /tmp/tmp.1c8DubzPGh /tmp/tmp.mL0hjyYyCF + return 0 + enable_pgBouncer some-name + local cluster_name=some-name + local replicas_num=1 + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.82JXnbPF2q ++ mktemp + local LAST_ERR=/tmp/tmp.KdaUEpVnUA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.82JXnbPF2q perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.KdaUEpVnUA + rm /tmp/tmp.82JXnbPF2q /tmp/tmp.KdaUEpVnUA + return 0 + sleep 60 + kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.99Rqw2Pmsi ++ mktemp + local LAST_ERR=/tmp/tmp.GoFwHD8EbA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.99Rqw2Pmsi deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.GoFwHD8EbA + rm /tmp/tmp.99Rqw2Pmsi /tmp/tmp.GoFwHD8EbA + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=upgrade-21023 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rNc1obJzLm +++ mktemp ++ local LAST_ERR=/tmp/tmp.lmQfWQIOcN ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.rNc1obJzLm ++ cat /tmp/tmp.lmQfWQIOcN ++ rm /tmp/tmp.rNc1obJzLm /tmp/tmp.lmQfWQIOcN ++ return 0 + status_query='kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iNYmijuSiw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Kv708Ni6qe +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade +++ kubectl -n upgrade-21023 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iNYmijuSiw +++ cat /tmp/tmp.Kv708Ni6qe +++ rm /tmp/tmp.iNYmijuSiw /tmp/tmp.Kv708Ni6qe +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + wait_deployment some-name + local name=some-name + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=upgrade-21023 + sleep 10 + set +o xtrace some-name-pgbouncer + sleep 10 + compare_psql select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 verify-ca + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local sslmode=verify-ca + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql + run_psql_ssl '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 postgres '' verify-ca + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.upgrade-21023 + local driver=postgres ++ kubectl_bin get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8DXHi3OK9m +++ mktemp ++ local LAST_ERR=/tmp/tmp.pV0AeccIJb ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.8DXHi3OK9m ++ cat /tmp/tmp.pV0AeccIJb ++ rm /tmp/tmp.8DXHi3OK9m /tmp/tmp.pV0AeccIJb ++ return 0 + local client_container=pg-client-7f96cb6f54-zqmrb + local sslmode=verify-ca + kubectl_bin exec pg-client-7f96cb6f54-zqmrb -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023' ++ mktemp + local LAST_OUT=/tmp/tmp.5Zqo1plE7M ++ mktemp + local LAST_ERR=/tmp/tmp.HDhZKQPEPb + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl exec pg-client-7f96cb6f54-zqmrb -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | PGSSLMODE=verify-ca PGSSLROOTCERT=/tmp/tls/ca.crt psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.upgrade-21023' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5Zqo1plE7M + cat /tmp/tmp.HDhZKQPEPb + rm /tmp/tmp.5Zqo1plE7M /tmp/tmp.HDhZKQPEPb + return 0 + '[' '!' -s /tmp/tmp.V8NoLLRtUg/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/upgrade/compare/select-1.sql /tmp/tmp.V8NoLLRtUg/select-1.sql + destroy upgrade-21023 + grep -v level=info ++ get_operator_pod + grep -v level=debug ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'Getting pods from source' + grep -v 'the object has been modified' + grep -v 'Getting tasks for pod' + grep -v 'get backup status: Job.batch' ++ local LAST_OUT=/tmp/tmp.VBOcFLgFGn + sort -u +++ mktemp + tee /tmp/tmp.V8NoLLRtUg/operator.log ++ local LAST_ERR=/tmp/tmp.qrZCYK33VS ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.VBOcFLgFGn ++ cat /tmp/tmp.qrZCYK33VS ++ rm /tmp/tmp.VBOcFLgFGn /tmp/tmp.qrZCYK33VS ++ return 0 + kubectl_bin logs postgres-operator-5d57849c8d-9qx2r -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.7ap4BEHGlX ++ mktemp + local LAST_ERR=/tmp/tmp.iiXoj6Be5g + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl logs postgres-operator-5d57849c8d-9qx2r -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7ap4BEHGlX + cat /tmp/tmp.iiXoj6Be5g + rm /tmp/tmp.7ap4BEHGlX /tmp/tmp.iiXoj6Be5g + return 0 ] { } }, ] }, } } { } }, ] { } }, ] ], { } }, }, { },{ }] }], { } }, , ], } }, }], } }, } }, { } ], } }, } }, }, { } }, 2024/05/24 18:10:28 INF 1 (localhost:4150) connecting to nsqd 2024/05/24 18:15:00 INF 2 (localhost:4150) connecting to nsqd 2024/05/24 18:15:25 INF 3 (localhost:4150) connecting to nsqd 2024/05/24 18:15:50 INF 4 (localhost:4150) connecting to nsqd 2024/05/24 18:17:27 INF 5 (localhost:4150) connecting to nsqd 2024/05/24 18:18:16 INF 6 (localhost:4150) connecting to nsqd 2024/05/24 18:18:22 INF 7 (localhost:4150) connecting to nsqd 2024/05/24 18:18:38 INF 8 (localhost:4150) connecting to nsqd "affinity": "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "backrest-command": "backup", "configMap": { "containerPort": 5432, "containers": [{ "cpu": "1", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "defaultMode": 288 "defaultMode": 288, "dnsPolicy": "ClusterFirst" "emptyDir": { "emptyDir": { "env": [{ "fieldPath": "metadata.namespace" "fieldRef": { "fsGroup": 2, I0524 18:07:13.629228 1 shared_informer.go:240] Waiting for caches to sync for namespace I0524 18:07:13.729712 1 shared_informer.go:247] Caches are synced for namespace I0524 18:07:13.936446 1 shared_informer.go:240] Waiting for caches to sync for upgrade-21023 I0524 18:07:14.036918 1 shared_informer.go:247] Caches are synced for upgrade-21023 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "imagePullPolicy": "IfNotPresent", "key": "crunchy-pgbouncer", "key": "password" "key": "pg-cluster", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "matchExpressions": [ "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory", "memory": "128Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/pgconf/", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/tmp", "mountPath": "/tmp", "name": "backrest", "name": "COMMAND", "name": "COMMAND_OPTS", "name": "MODE", "name": "NAMESPACE", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "PODNAME", "name": "postgres", "name": "some-name-full-sch-backup", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer" "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer-cm" "name": "some-name-pgbouncer-secret", "name": "some-name-pgbouncer-secret", "name": "some-name-ssl-ca" "name": "some-name-ssl-keypair" "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "nodePort": 0 "operator": "Exists" "operator": "In", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name", "pg-cluster": "some-name", "pg-cluster": "some-name", "pgo-backrest-job": "true", "pgo-backrest": "true", "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "port": 5432, "ports": [{ "ports": [ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "protocol": "TCP", "protocol": "TCP", "readOnly": false "readOnlyRootFilesystem": true "replicas": 1, "requests": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secretKeyRef": { "securityContext": { "securityContext": { "securityContext": {}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer" "sessionAffinity": "None" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "some-name" "sources": [ "spec": { "spec": { "spec": { "strategy": { "targetPort": 5432, "template": { time="2024-05-24T18:07:13Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.7.0 time="2024-05-24T18:07:14Z" level=error msg="create pgcluster resource: create pgcluster resource: pgclusters.pg.percona.com \"some-name\" already exists" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgc.(*Controller).onAdd()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgc/pgc.go:174" version=1.7.0 time="2024-05-24T18:16:14Z" level=error msg="-{\"callbacks\":{\"on_role_change\":\"/opt/crunchy/bin/postgres-ha/callbacks/pgha-on-role-change.sh\"},\"create_replica_methods\":[\"pgbackrest\",\"basebackup\"],\"pg_hba\":[\"local all postgres peer\",\"hostssl replication primaryuser 0.0.0.0/0 md5\",\"hostssl all primaryuser 0.0.0.0/0 reject\",\"hostssl all all 0.0.0.0/0 md5\"],\"pgbackrest\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh replica\",\"keep_data\":true,\"no_params\":true},\"pgbackrest_standby\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh standby\",\"keep_data\":true,\"no_master\":1,\"no_params\":true},\"pgpass\":\"/tmp/.pgpass\",\"remove_data_directory_on_rewind_failure\":true,\"use_unix_socket\":true}\n+{\"authentication\":{\"replication\":{\"sslcert\":\"/pgconf/tls-replication/tls.crt\",\"sslkey\":\"/pgconf/tls-replication/tls.key\",\"sslmode\":\"verify-ca\",\"sslrootcert\":\"/pgconf/tls/ca.crt\"}},\"callbacks\":{\"on_role_change\":\"/opt/crunchy/bin/postgres-ha/callbacks/pgha-on-role-change.sh\"},\"create_replica_methods\":[\"pgbackrest\",\"basebackup\"],\"pg_hba\":[\"local all postgres peer\",\"hostssl replication primaryuser 0.0.0.0/0 cert\",\"hostssl all primaryuser 0.0.0.0/0 reject\",\"hostssl all all 0.0.0.0/0 md5\"],\"pgbackrest\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh replica\",\"keep_data\":true,\"no_params\":true},\"pgbackrest_standby\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh standby\",\"keep_data\":true,\"no_master\":1,\"no_params\":true},\"pgpass\":\"/tmp/.pgpass\",\"remove_data_directory_on_rewind_failure\":true,\"use_unix_socket\":true}\n\x1b[0;32mFri May 24 18:16:14 UTC 2024 INFO: Reload Config: Detected config change, reloading local configuration\x1b[0m\n\x1b[0;31mFri May 24 18:16:14 UTC 2024 ERROR: Error reloading local config: Unable to reload local configuration\x1b[0m\n" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.7.0 time="2024-05-24T18:16:14Z" level=error msg="command terminated with exit code 7" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T18:16:14Z" level=error msg="command terminated with exit code 7" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.7.0 time="2024-05-24T18:17:26Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T18:17:26Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.7.0 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "backup" "value": "false" "valueFrom": { "value": "pgbackrest" "value": "posix" "values": [ "value": "some-name-backrest-shared-repo-6f8f947dbc-lsm8w" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.101.81.21 --db-path=/pgdata/some-name" "value": "/tmp" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumes": [ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.vQSjEpyxnS ++ mktemp + local LAST_ERR=/tmp/tmp.e6Wb5s95DS + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.vQSjEpyxnS + cat /tmp/tmp.e6Wb5s95DS Error from server (NotFound): error when deleting "/tmp/tmp.V8NoLLRtUg/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.V8NoLLRtUg/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.V8NoLLRtUg/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.V8NoLLRtUg/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.V8NoLLRtUg/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.vQSjEpyxnS /tmp/tmp.e6Wb5s95DS + return 1 + true + yq w -d4 /tmp/tmp.V8NoLLRtUg/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.V8NoLLRtUg/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.wt0PUTldUm ++ mktemp + local LAST_ERR=/tmp/tmp.zexTdU0rmM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl apply -f /tmp/tmp.V8NoLLRtUg/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.wt0PUTldUm serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.zexTdU0rmM + rm /tmp/tmp.wt0PUTldUm /tmp/tmp.zexTdU0rmM + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=upgrade-21023 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n upgrade-21023 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.V8NoLLRtUg/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.rUihd1Zq83 ++ mktemp + local LAST_ERR=/tmp/tmp.JFrRx2Cycu + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete -f /tmp/tmp.V8NoLLRtUg/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.rUihd1Zq83 serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.JFrRx2Cycu + rm /tmp/tmp.rUihd1Zq83 /tmp/tmp.JFrRx2Cycu + return 0 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' ++ mktemp + xargs kubectl delete ns --grace-period=0 --force=true + local LAST_OUT=/tmp/tmp.KrMnqoiO9P ++ mktemp + local LAST_ERR=/tmp/tmp.9LA71eyrjl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KrMnqoiO9P + cat /tmp/tmp.9LA71eyrjl + rm /tmp/tmp.KrMnqoiO9P /tmp/tmp.9LA71eyrjl + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "upgrade-21023" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.BDocn4MaKV ++ mktemp + local LAST_ERR=/tmp/tmp.kNSpTLyolr + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.BDocn4MaKV + cat /tmp/tmp.kNSpTLyolr Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.BDocn4MaKV /tmp/tmp.kNSpTLyolr + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.ODoIswK43R ++ mktemp + local LAST_ERR=/tmp/tmp.WP0RDfHB1J + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-upgrade + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.ODoIswK43R + cat /tmp/tmp.WP0RDfHB1J Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.ODoIswK43R /tmp/tmp.WP0RDfHB1J + return 1 + true + rm -rf /tmp/tmp.V8NoLLRtUg