++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/recreate.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/recreate.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ kubectl version -o json +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.15.1 ++ '[' v3 == v2 ']' + create_namespace recreate-18368 + local namespace=recreate-18368 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + cleanup_rbac + awk '{print$1}' ++ mktemp + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.BwDTTVqqi2 ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.xs8C8CTUug + local LAST_ERR=/tmp/tmp.w85qsDMFpa + local exit_status=0 + local timeout=4 + set +o errexit ++ mktemp ++ seq 0 2 + local LAST_ERR=/tmp/tmp.5gPaUt9DES + local exit_status=0 + local timeout=4 + set +o errexit + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get ns ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BwDTTVqqi2 + cat /tmp/tmp.w85qsDMFpa + rm /tmp/tmp.BwDTTVqqi2 /tmp/tmp.w85qsDMFpa + return 0 error: resource(s) were provided, but no name was specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.xs8C8CTUug + cat /tmp/tmp.5gPaUt9DES Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.xs8C8CTUug /tmp/tmp.5gPaUt9DES + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.cIsfxCS5km ++ mktemp + local LAST_ERR=/tmp/tmp.fLYEfZlcEi + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.cIsfxCS5km + cat /tmp/tmp.fLYEfZlcEi Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.cIsfxCS5km /tmp/tmp.fLYEfZlcEi + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace recreate-18368 ++ mktemp + local LAST_OUT=/tmp/tmp.5O3vCRSHJ3 ++ mktemp + local LAST_ERR=/tmp/tmp.IPeag02n49 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete namespace recreate-18368 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete namespace recreate-18368 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete namespace recreate-18368 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.5O3vCRSHJ3 + cat /tmp/tmp.IPeag02n49 Error from server (NotFound): namespaces "recreate-18368" not found + rm /tmp/tmp.5O3vCRSHJ3 /tmp/tmp.IPeag02n49 + return 1 + : + wait_for_delete namespace/recreate-18368 + local res=namespace/recreate-18368 + set +o xtrace namespace/recreate-18368 - Error from server (NotFound): namespaces "recreate-18368" not found + kubectl_bin create namespace recreate-18368 ++ mktemp + local LAST_OUT=/tmp/tmp.SDIGug7FBh ++ mktemp + local LAST_ERR=/tmp/tmp.x5bFC9faJN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl create namespace recreate-18368 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.SDIGug7FBh namespace/recreate-18368 created + cat /tmp/tmp.x5bFC9faJN + rm /tmp/tmp.SDIGug7FBh /tmp/tmp.x5bFC9faJN + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.CF9e03bpOa +++ mktemp ++ local LAST_ERR=/tmp/tmp.NoK0X095Ay ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.CF9e03bpOa ++ cat /tmp/tmp.NoK0X095Ay ++ rm /tmp/tmp.CF9e03bpOa /tmp/tmp.NoK0X095Ay ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-sandbox --namespace=recreate-18368 ++ mktemp + local LAST_OUT=/tmp/tmp.MqmUXDdke9 ++ mktemp + local LAST_ERR=/tmp/tmp.aREK6nZpSn + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-sandbox --namespace=recreate-18368 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.MqmUXDdke9 Context "gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-sandbox" modified. + cat /tmp/tmp.aREK6nZpSn + rm /tmp/tmp.MqmUXDdke9 /tmp/tmp.aREK6nZpSn + return 0 + deploy_operator + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local action=install + local namespace=recreate-18368 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml disabled install recreate-18368 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=install + local namespace=recreate-18368 + local dont_send_telemetry=false + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml 'data[values.yaml]' + /usr/bin/sed -e 's#^namespace: .*#namespace: "recreate-18368"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "recreate-18368"#g' + cat /dev/fd/63 - + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 ++ echo 'crunchy_debug: "true"' ++ cut -d: -f2 ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 ++ cut -d: -f1 + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-776-0087ef129"#g' + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml metadata.namespace recreate-18368 + yq w -d3 - 'subjects[0].namespace' recreate-18368 + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-deployer + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install ++ cat /tmp/tmp.XJtLFBxkzn/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "recreate-18368" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-776-0087ef129" pgo_operator_namespace: "recreate-18368"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.XJtLFBxkzn/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.v0VXYXB3wR ++ mktemp + local LAST_ERR=/tmp/tmp.aSQo69O0As + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl apply -f /tmp/tmp.XJtLFBxkzn/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.v0VXYXB3wR serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.aSQo69O0As + rm /tmp/tmp.v0VXYXB3wR /tmp/tmp.aSQo69O0As + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=recreate-18368 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.qiX3FnGeRU ++ mktemp + local LAST_ERR=/tmp/tmp.08GbflaQzA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qiX3FnGeRU serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.08GbflaQzA + rm /tmp/tmp.qiX3FnGeRU /tmp/tmp.08GbflaQzA + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FEarQnW0qM +++ mktemp ++ local LAST_ERR=/tmp/tmp.DPSxlnVB5d ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FEarQnW0qM ++ cat /tmp/tmp.DPSxlnVB5d ++ rm /tmp/tmp.FEarQnW0qM /tmp/tmp.DPSxlnVB5d ++ return 0 + wait_pod postgres-operator-c6cccc8b9-gr4rq + local pod=postgres-operator-c6cccc8b9-gr4rq + local ns= + set +o xtrace postgres-operator-c6cccc8b9-gr4rqtrue + [[ -n '' ]] + cluster=some-name + spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local cluster=some-name + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local backup=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=recreate-18368 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name recreate-18368 + local cluster=some-name + local namespace=recreate-18368 + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n recreate-18368 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.1xLO8YmbfU ++ mktemp + local LAST_ERR=/tmp/tmp.UghaHPrjcr + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1xLO8YmbfU secret/some-name-users-secret created + cat /tmp/tmp.UghaHPrjcr + rm /tmp/tmp.1xLO8YmbfU /tmp/tmp.UghaHPrjcr + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n recreate-18368 apply -f - + yq w - metadata.name some-name-pmm-secret ++ mktemp + local LAST_OUT=/tmp/tmp.5nFbWpK4H3 ++ mktemp + local LAST_ERR=/tmp/tmp.VYnpodrFlw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5nFbWpK4H3 secret/some-name-pmm-secret created + cat /tmp/tmp.VYnpodrFlw + rm /tmp/tmp.5nFbWpK4H3 /tmp/tmp.VYnpodrFlw + return 0 + kubectl_bin -n recreate-18368 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.ytOccIsVCM ++ mktemp + local LAST_ERR=/tmp/tmp.OteblBFh3c + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ytOccIsVCM deployment.apps/pg-client created + cat /tmp/tmp.OteblBFh3c + rm /tmp/tmp.ytOccIsVCM /tmp/tmp.OteblBFh3c + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name false false false false false false false false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity=false + local schedule=false + local schedule_type=false + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - metadata.labels.deployment-name some-name + yq w - metadata.labels.name some-name + yq w - spec.clustername some-name + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.name some-name + yq w - spec.database some-name + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.keepBackups false + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.user some-name + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.secretsName some-name-users-secret + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.keepData false + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.XJtLFBxkzn/cr.yaml -n recreate-18368 ++ mktemp + local LAST_OUT=/tmp/tmp.vRdZ9pB604 ++ mktemp + local LAST_ERR=/tmp/tmp.s06VnhJKzM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl apply -f /tmp/tmp.XJtLFBxkzn/cr.yaml -n recreate-18368 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.vRdZ9pB604 perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.s06VnhJKzM + rm /tmp/tmp.vRdZ9pB604 /tmp/tmp.s06VnhJKzM + return 0 + wait_deployment some-name-backrest-shared-repo + local name=some-name-backrest-shared-repo + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-backrest-shared-repo.. + wait_deployment some-name + local name=some-name + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name......... + [[ false == \g\c\s\+* ]] + wait_job_completion some-name-stanza-create false + local job=some-name-stanza-create + local mandatory=false + local ns=recreate-18368 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' some-name-stanza-create+ retry=0 ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name + local job=backrest-backup-some-name + local mandatory=true + local ns=recreate-18368 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' backrest-backup-some-name+ retry=0 ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=recreate-18368 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ZQfxAiZWnL +++ mktemp ++ local LAST_ERR=/tmp/tmp.HRcZ24oTFQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ZQfxAiZWnL ++ cat /tmp/tmp.HRcZ24oTFQ ++ rm /tmp/tmp.ZQfxAiZWnL /tmp/tmp.HRcZ24oTFQ ++ return 0 + status_query='kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.sPJa3tvDcm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SqgJBlr8Y4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.sPJa3tvDcm +++ cat /tmp/tmp.SqgJBlr8Y4 +++ rm /tmp/tmp.sPJa3tvDcm /tmp/tmp.SqgJBlr8Y4 +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4JRBWZ9ePs ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PzKpjujJR8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.4JRBWZ9ePs +++ cat /tmp/tmp.PzKpjujJR8 +++ rm /tmp/tmp.4JRBWZ9ePs /tmp/tmp.PzKpjujJR8 +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JkA2w6EiRy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bEd5M7ZWqT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JkA2w6EiRy +++ cat /tmp/tmp.bEd5M7ZWqT +++ rm /tmp/tmp.JkA2w6EiRy /tmp/tmp.bEd5M7ZWqT +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zOvbd6GXO8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YYUo52xmqU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zOvbd6GXO8 +++ cat /tmp/tmp.YYUo52xmqU +++ rm /tmp/tmp.zOvbd6GXO8 /tmp/tmp.YYUo52xmqU +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.W3OghuoIy5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.d0zizaJdGj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.W3OghuoIy5 +++ cat /tmp/tmp.d0zizaJdGj +++ rm /tmp/tmp.W3OghuoIy5 /tmp/tmp.d0zizaJdGj +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.AqBDLpLPtf +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.lDw2DvrPO3 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.AqBDLpLPtf ++++ cat /tmp/tmp.lDw2DvrPO3 ++++ rm /tmp/tmp.AqBDLpLPtf /tmp/tmp.lDw2DvrPO3 ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gw7FXZj7lf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GF1m2jlIyV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gw7FXZj7lf +++ cat /tmp/tmp.GF1m2jlIyV +++ rm /tmp/tmp.gw7FXZj7lf /tmp/tmp.GF1m2jlIyV +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.recreate-18368 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1But7IrJRo +++ mktemp ++ local LAST_ERR=/tmp/tmp.2kEJdcyINU ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.1But7IrJRo ++ cat /tmp/tmp.2kEJdcyINU ++ rm /tmp/tmp.1But7IrJRo /tmp/tmp.2kEJdcyINU ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.6mwCpsr5K1 ++ mktemp + local LAST_ERR=/tmp/tmp.98muRx7gGP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.6mwCpsr5K1 + cat /tmp/tmp.98muRx7gGP + rm /tmp/tmp.6mwCpsr5K1 /tmp/tmp.98muRx7gGP + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HU1c0sjaiU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kIQDElsepu +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.HU1c0sjaiU +++ cat /tmp/tmp.kIQDElsepu +++ rm /tmp/tmp.HU1c0sjaiU /tmp/tmp.kIQDElsepu +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.54mjB4Supk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0BXQjkwWq2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.54mjB4Supk +++ cat /tmp/tmp.0BXQjkwWq2 +++ rm /tmp/tmp.54mjB4Supk /tmp/tmp.0BXQjkwWq2 +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BA9YYtLwBF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yiL47Z1QkS +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BA9YYtLwBF +++ cat /tmp/tmp.yiL47Z1QkS +++ rm /tmp/tmp.BA9YYtLwBF /tmp/tmp.yiL47Z1QkS +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.43z9aDFkBO +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2HZM4RdfXq ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.43z9aDFkBO ++++ cat /tmp/tmp.2HZM4RdfXq ++++ rm /tmp/tmp.43z9aDFkBO /tmp/tmp.2HZM4RdfXq ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.y8ZxaIQ3Na ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aQSPlm89Lv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.y8ZxaIQ3Na +++ cat /tmp/tmp.aQSPlm89Lv +++ rm /tmp/tmp.y8ZxaIQ3Na /tmp/tmp.aQSPlm89Lv +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.recreate-18368 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GPBIlfoyvM +++ mktemp ++ local LAST_ERR=/tmp/tmp.9XP68RCIZd ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GPBIlfoyvM ++ cat /tmp/tmp.9XP68RCIZd ++ rm /tmp/tmp.GPBIlfoyvM /tmp/tmp.9XP68RCIZd ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.tFIHjVJXYp ++ mktemp + local LAST_ERR=/tmp/tmp.TvNomm3DKV + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.tFIHjVJXYp + cat /tmp/tmp.TvNomm3DKV + rm /tmp/tmp.tFIHjVJXYp /tmp/tmp.TvNomm3DKV + return 0 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6iDisCEho9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LYxYilaD0I +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6iDisCEho9 +++ cat /tmp/tmp.LYxYilaD0I +++ rm /tmp/tmp.6iDisCEho9 /tmp/tmp.LYxYilaD0I +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cbT5vdVXfH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VtlMoeeiqH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cbT5vdVXfH +++ cat /tmp/tmp.VtlMoeeiqH +++ rm /tmp/tmp.cbT5vdVXfH /tmp/tmp.VtlMoeeiqH +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TiPf0gSphx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ulBou4T0xt +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TiPf0gSphx +++ cat /tmp/tmp.ulBou4T0xt +++ rm /tmp/tmp.TiPf0gSphx /tmp/tmp.ulBou4T0xt +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gT5Pl7whVz +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.MvCDMkR73j ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.gT5Pl7whVz ++++ cat /tmp/tmp.MvCDMkR73j ++++ rm /tmp/tmp.gT5Pl7whVz /tmp/tmp.MvCDMkR73j ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.nKoPBqZtNT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8eHtcc99qn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.nKoPBqZtNT +++ cat /tmp/tmp.8eHtcc99qn +++ rm /tmp/tmp.nKoPBqZtNT /tmp/tmp.8eHtcc99qn +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.recreate-18368 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.p4Dj5rnJsy +++ mktemp ++ local LAST_ERR=/tmp/tmp.AUBcxsfJIb ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.p4Dj5rnJsy ++ cat /tmp/tmp.AUBcxsfJIb ++ rm /tmp/tmp.p4Dj5rnJsy /tmp/tmp.AUBcxsfJIb ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.KBNOZ7hGKF ++ mktemp + local LAST_ERR=/tmp/tmp.oDtmoNxAX1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KBNOZ7hGKF + cat /tmp/tmp.oDtmoNxAX1 + rm /tmp/tmp.KBNOZ7hGKF /tmp/tmp.oDtmoNxAX1 + return 0 + sleep 10 + desc 'Quick pgbouncer switch off and switch on' + set +o xtrace ----------------------------------------------------------------------------------- Quick pgbouncer switch off and switch on ----------------------------------------------------------------------------------- + disable_pgBouncer some-name + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.xYoDUw4Msp ++ mktemp + local LAST_ERR=/tmp/tmp.3QSaYBEBL7 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xYoDUw4Msp perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.3QSaYBEBL7 + rm /tmp/tmp.xYoDUw4Msp /tmp/tmp.3QSaYBEBL7 + return 0 + kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.auDlvyiXjR ++ mktemp + local LAST_ERR=/tmp/tmp.kSWQhDHm6r + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.auDlvyiXjR deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.kSWQhDHm6r + rm /tmp/tmp.auDlvyiXjR /tmp/tmp.kSWQhDHm6r + return 0 + enable_pgBouncer some-name 3 + local cluster_name=some-name + local replicas_num=3 + kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.ZcXeC51DNX ++ mktemp + local LAST_ERR=/tmp/tmp.D4aDT5EFX8 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ZcXeC51DNX perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.D4aDT5EFX8 + rm /tmp/tmp.ZcXeC51DNX /tmp/tmp.D4aDT5EFX8 + return 0 + sleep 60 + kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.ZbAjUpQmjb ++ mktemp + local LAST_ERR=/tmp/tmp.K6R3H4v34x + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ZbAjUpQmjb deployment.apps/some-name-pgbouncer condition met + cat /tmp/tmp.K6R3H4v34x + rm /tmp/tmp.ZbAjUpQmjb /tmp/tmp.K6R3H4v34x + return 0 + wait_deployment some-name-pgbouncer + local name=some-name-pgbouncer + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-pgbouncer ++ get_psql_user_pass pgbouncer some-name ++ local user=pgbouncer ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.wFuVPteI2I ++++ mktemp +++ local LAST_ERR=/tmp/tmp.l6ZxGl2rJ4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.wFuVPteI2I +++ cat /tmp/tmp.l6ZxGl2rJ4 +++ rm /tmp/tmp.wFuVPteI2I /tmp/tmp.l6ZxGl2rJ4 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cB9lHshuPi ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WA3rnh7Ye3 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cB9lHshuPi +++ cat /tmp/tmp.WA3rnh7Ye3 +++ rm /tmp/tmp.cB9lHshuPi /tmp/tmp.WA3rnh7Ye3 +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RzWGu4Svod ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W5sjR148SB +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RzWGu4Svod +++ cat /tmp/tmp.W5sjR148SB +++ rm /tmp/tmp.RzWGu4Svod /tmp/tmp.W5sjR148SB +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.OgpowbhUQy +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.NJSXozGWEn ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.OgpowbhUQy ++++ cat /tmp/tmp.NJSXozGWEn ++++ rm /tmp/tmp.OgpowbhUQy /tmp/tmp.NJSXozGWEn ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.C7TqiBYaRx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.BAgWokUk7d +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.C7TqiBYaRx +++ cat /tmp/tmp.BAgWokUk7d +++ rm /tmp/tmp.C7TqiBYaRx /tmp/tmp.BAgWokUk7d +++ return 0 ++ echo pgbouncer_password + compare_psql_cmd show-1 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368 + local command_id=show-1 + local 'command=SHOW DATABASES' + local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/show-1.sql + run_psql 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368 postgres '' + local 'command=SHOW DATABASES' + local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ogBL4zFjXH +++ mktemp ++ local LAST_ERR=/tmp/tmp.tVG7RzCM09 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ogBL4zFjXH ++ cat /tmp/tmp.tVG7RzCM09 ++ rm /tmp/tmp.ogBL4zFjXH /tmp/tmp.tVG7RzCM09 ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.lhpJHAa40K ++ mktemp + local LAST_ERR=/tmp/tmp.NFyGX0h21P + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.lhpJHAa40K + cat /tmp/tmp.NFyGX0h21P + rm /tmp/tmp.lhpJHAa40K /tmp/tmp.NFyGX0h21P + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/show-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/show-1.sql /tmp/tmp.XJtLFBxkzn/show-1.sql ++ get_psql_user_pass some-name some-name ++ local user=some-name ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iFh9Q949a7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LVnUNGTqKe +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iFh9Q949a7 +++ cat /tmp/tmp.LVnUNGTqKe +++ rm /tmp/tmp.iFh9Q949a7 /tmp/tmp.LVnUNGTqKe +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aSP26XhANX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MnBJkFyaPE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.aSP26XhANX +++ cat /tmp/tmp.MnBJkFyaPE +++ rm /tmp/tmp.aSP26XhANX /tmp/tmp.MnBJkFyaPE +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jsUby8Uui8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2Bm5N2o0NS +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.jsUby8Uui8 +++ cat /tmp/tmp.2Bm5N2o0NS +++ rm /tmp/tmp.jsUby8Uui8 /tmp/tmp.2Bm5N2o0NS +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.eagSOF9J6Q +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.AbIVJPbmVt ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.eagSOF9J6Q ++++ cat /tmp/tmp.AbIVJPbmVt ++++ rm /tmp/tmp.eagSOF9J6Q /tmp/tmp.AbIVJPbmVt ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.cFKEJ5OjOq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rrfpUFn88Y +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.cFKEJ5OjOq +++ cat /tmp/tmp.rrfpUFn88Y +++ rm /tmp/tmp.cFKEJ5OjOq /tmp/tmp.rrfpUFn88Y +++ return 0 ++ echo some-name_pass + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-18368 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-18368 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-18368 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.T06cZAiOJU +++ mktemp ++ local LAST_ERR=/tmp/tmp.0lf09YN8KG ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.T06cZAiOJU ++ cat /tmp/tmp.0lf09YN8KG ++ rm /tmp/tmp.T06cZAiOJU /tmp/tmp.0lf09YN8KG ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.yr24fPh5KV ++ mktemp + local LAST_ERR=/tmp/tmp.AwiFy3Jkea + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yr24fPh5KV + cat /tmp/tmp.AwiFy3Jkea + rm /tmp/tmp.yr24fPh5KV /tmp/tmp.AwiFy3Jkea + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.XJtLFBxkzn/select-1.sql + compare_kubectl deployment/some-name-pgbouncer + local resource=deployment/some-name-pgbouncer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml + local new_result=/tmp/tmp.XJtLFBxkzn/deployment_some-name-pgbouncer.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml deployment/some-name-pgbouncer + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.uid' + yq d - metadata.selfLink ++ mktemp + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + local LAST_OUT=/tmp/tmp.F29HPSBK8t ++ mktemp + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + local LAST_ERR=/tmp/tmp.d0t57zzXBK + local exit_status=0 + local timeout=4 + set +o errexit + yq d - '**.clusterIPs' + yq d - '**.creationTimestamp' + yq d - '**.procMount' + yq d - '**.finalizers' + yq d - '**.image' + yq d - '**.clusterIP' + yq d - 'metadata.annotations."cloud.google.com/neg"' ++ seq 0 2 + yq d - '**."kubernetes.io/pvc-protection"' + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml deployment/some-name-pgbouncer + yq d - metadata.deletionTimestamp + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - metadata.resourceVersion + yq d - '**.namespace' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**."percona.com/*"' + yq d - spec.volumeMode + yq d - '**.nodePort' + yq d - '**.dataSource' + yq d - '**.storageClassName' + yq d - spec.nodeName + yq d - '**.imagePullSecrets' + yq d - '**.volumeName' + yq d - spec.podReplacementPolicy + yq d - '**.enableServiceLinks' + yq d - spec.manualSelector + yq d - '**.(name==suffix)' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.controller-uid' + yq d - '**.(name==PODNAME)' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.preemptionPolicy' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**.healthCheckNodePort' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - status + yq d - '**.(name==NAMESPACE)' + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**.internalTrafficPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.F29HPSBK8t + cat /tmp/tmp.d0t57zzXBK + rm /tmp/tmp.F29HPSBK8t /tmp/tmp.d0t57zzXBK + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/deployment_some-name-pgbouncer.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml /tmp/tmp.XJtLFBxkzn/deployment_some-name-pgbouncer.yml + compare_kubectl service/some-name-pgbouncer + local resource=service/some-name-pgbouncer + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml + local new_result=/tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - metadata.resourceVersion + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.clusterIPs' + yq d - '**.creationTimestamp' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - '**.dataSource' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.finalizers' + yq d - '**.volumeName' + yq d - spec.volumeMode + yq d - spec.nodeName + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - spec.podReplacementPolicy + yq d - '**.enableServiceLinks' + yq d - spec.manualSelector + yq d - status + yq d - '**.(name==suffix)' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**."percona.com/*"' + yq d - '**.healthCheckNodePort' + yq d - '**.ipFamilies' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.internalTrafficPolicy' + yq d - '**.clusterIP' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**.procMount' + yq d - '**.storageClassName' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.imagePullSecrets' + yq d - '**.(name==NAMESPACE)' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.ipFamilyPolicy' + yq d - '**.controller-uid' + yq d - '**.nodePort' + yq d - '**.preemptionPolicy' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - '**.(name==PODNAME)' + yq d - '**.image' + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - metadata.deletionTimestamp + yq d - metadata.selfLink + kubectl_bin get -o yaml service/some-name-pgbouncer ++ mktemp + local LAST_OUT=/tmp/tmp.oEDg2ofO21 ++ mktemp + local LAST_ERR=/tmp/tmp.xntoOSKlIi + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml service/some-name-pgbouncer + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.oEDg2ofO21 + cat /tmp/tmp.xntoOSKlIi + rm /tmp/tmp.oEDg2ofO21 /tmp/tmp.xntoOSKlIi + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml kind ++ yq r /tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml metadata.annotations + [[ x{} == \x ]] ++ yq r /tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml metadata.annotations + [[ x{} == \x\{\} ]] + yq d -i /tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml metadata.annotations + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml /tmp/tmp.XJtLFBxkzn/service_some-name-pgbouncer.yml ++ seq 1 3 + for repl in '$(seq 1 3)' + desc 'check replica 1' + set +o xtrace ----------------------------------------------------------------------------------- check replica 1 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.aXamhwPUzP ++ mktemp + local LAST_ERR=/tmp/tmp.7CLWLlLduH + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.aXamhwPUzP perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.7CLWLlLduH + rm /tmp/tmp.aXamhwPUzP /tmp/tmp.7CLWLlLduH + return 0 + sleep 30 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl1 + sleep 30 + check_replica some-name-repl1 some-name select-1 + local name=some-name-repl1 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.95zFNlWEGZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5DFy295yTi +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.95zFNlWEGZ +++ cat /tmp/tmp.5DFy295yTi +++ rm /tmp/tmp.95zFNlWEGZ /tmp/tmp.5DFy295yTi +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.en7yhsppZq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5h1SYq2L5T +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.en7yhsppZq +++ cat /tmp/tmp.5h1SYq2L5T +++ rm /tmp/tmp.en7yhsppZq /tmp/tmp.5h1SYq2L5T +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.GnmkECxJe5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4ObXhcGWwI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.GnmkECxJe5 +++ cat /tmp/tmp.4ObXhcGWwI +++ rm /tmp/tmp.GnmkECxJe5 /tmp/tmp.4ObXhcGWwI +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.RFhmycPzej +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.uXzBim3vIR ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.RFhmycPzej ++++ cat /tmp/tmp.uXzBim3vIR ++++ rm /tmp/tmp.RFhmycPzej /tmp/tmp.uXzBim3vIR ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YL4UrceA9w ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TMovkiMPDM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.YL4UrceA9w +++ cat /tmp/tmp.TMovkiMPDM +++ rm /tmp/tmp.YL4UrceA9w /tmp/tmp.TMovkiMPDM +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IOh5K2zRVJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.sYNzzm2StM ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.IOh5K2zRVJ ++ cat /tmp/tmp.sYNzzm2StM ++ rm /tmp/tmp.IOh5K2zRVJ /tmp/tmp.sYNzzm2StM ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.YZUze7ulIN ++ mktemp + local LAST_ERR=/tmp/tmp.DRuKI0qgGf + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.YZUze7ulIN + cat /tmp/tmp.DRuKI0qgGf + rm /tmp/tmp.YZUze7ulIN /tmp/tmp.DRuKI0qgGf + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.6Rx8YFPb80 +++ mktemp ++ local LAST_ERR=/tmp/tmp.LqHGvsskAn ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.6Rx8YFPb80 ++ cat /tmp/tmp.LqHGvsskAn ++ rm /tmp/tmp.6Rx8YFPb80 /tmp/tmp.LqHGvsskAn ++ return 0 + replica_pod_name=some-name-repl1-5489f574fb-kppgm ++ grep some-name-repl1-5489f574fb-kppgm /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl1-5489f574fb-kppgm | 10.87.33.47 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.RHaYDajkMp +++ mktemp ++ local LAST_ERR=/tmp/tmp.vpW0tTbs79 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.RHaYDajkMp ++ cat /tmp/tmp.vpW0tTbs79 ++ rm /tmp/tmp.RHaYDajkMp /tmp/tmp.vpW0tTbs79 ++ return 0 + replica_pod_IP=10.87.33.47 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.33.47 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.33.47 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.33.47 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.33.47 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.975S9qtJF0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.p7RCppCMc1 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.975S9qtJF0 ++ cat /tmp/tmp.p7RCppCMc1 ++ rm /tmp/tmp.975S9qtJF0 /tmp/tmp.p7RCppCMc1 ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.33.47 ' ++ mktemp + local LAST_OUT=/tmp/tmp.HuFSOu8xhS ++ mktemp + local LAST_ERR=/tmp/tmp.uwLbD2PrDs + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.33.47 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.HuFSOu8xhS + cat /tmp/tmp.uwLbD2PrDs + rm /tmp/tmp.HuFSOu8xhS /tmp/tmp.uwLbD2PrDs + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.XJtLFBxkzn/select-1.sql + for repl in '$(seq 1 3)' + desc 'check replica 2' + set +o xtrace ----------------------------------------------------------------------------------- check replica 2 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.SdXqRatPEh ++ mktemp + local LAST_ERR=/tmp/tmp.i2qO2GpjvZ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.SdXqRatPEh perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.i2qO2GpjvZ + rm /tmp/tmp.SdXqRatPEh /tmp/tmp.i2qO2GpjvZ + return 0 + sleep 30 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl2 + sleep 30 + check_replica some-name-repl2 some-name select-1 + local name=some-name-repl2 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XRr3ppLshM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ofLcyY1OHK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XRr3ppLshM +++ cat /tmp/tmp.ofLcyY1OHK +++ rm /tmp/tmp.XRr3ppLshM /tmp/tmp.ofLcyY1OHK +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RyWvFB29JF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.7jL0aBf09a +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RyWvFB29JF +++ cat /tmp/tmp.7jL0aBf09a +++ rm /tmp/tmp.RyWvFB29JF /tmp/tmp.7jL0aBf09a +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Skhm4eiwmD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.p45TxQTJLY +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Skhm4eiwmD +++ cat /tmp/tmp.p45TxQTJLY +++ rm /tmp/tmp.Skhm4eiwmD /tmp/tmp.p45TxQTJLY +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.q1O3KouKYO +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.3FuwBqqUcD ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.q1O3KouKYO ++++ cat /tmp/tmp.3FuwBqqUcD ++++ rm /tmp/tmp.q1O3KouKYO /tmp/tmp.3FuwBqqUcD ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BNBNLou07R ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UbRubQYaFM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BNBNLou07R +++ cat /tmp/tmp.UbRubQYaFM +++ rm /tmp/tmp.BNBNLou07R /tmp/tmp.UbRubQYaFM +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bD5nMaxfxH +++ mktemp ++ local LAST_ERR=/tmp/tmp.zSOJEK5Wlq ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.bD5nMaxfxH ++ cat /tmp/tmp.zSOJEK5Wlq ++ rm /tmp/tmp.bD5nMaxfxH /tmp/tmp.zSOJEK5Wlq ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.oCRS3JIU0o ++ mktemp + local LAST_ERR=/tmp/tmp.0cArCNjqqd + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.oCRS3JIU0o + cat /tmp/tmp.0cArCNjqqd + rm /tmp/tmp.oCRS3JIU0o /tmp/tmp.0cArCNjqqd + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.bwitAky9qH +++ mktemp ++ local LAST_ERR=/tmp/tmp.FKOT9hrOuE ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.bwitAky9qH ++ cat /tmp/tmp.FKOT9hrOuE ++ rm /tmp/tmp.bwitAky9qH /tmp/tmp.FKOT9hrOuE ++ return 0 + replica_pod_name=some-name-repl2-759b4f7fdd-5g66w ++ grep some-name-repl2-759b4f7fdd-5g66w /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl2-759b4f7fdd-5g66w | 10.87.34.61 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ieBZYcAlOU +++ mktemp ++ local LAST_ERR=/tmp/tmp.9WVEpx7Lnw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ieBZYcAlOU ++ cat /tmp/tmp.9WVEpx7Lnw ++ rm /tmp/tmp.ieBZYcAlOU /tmp/tmp.9WVEpx7Lnw ++ return 0 + replica_pod_IP=10.87.34.61 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.34.61 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.34.61 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.34.61 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.34.61 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xXnuznamT0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.vIqLjVoQbz ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xXnuznamT0 ++ cat /tmp/tmp.vIqLjVoQbz ++ rm /tmp/tmp.xXnuznamT0 /tmp/tmp.vIqLjVoQbz ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.34.61 ' ++ mktemp + local LAST_OUT=/tmp/tmp.DAHk9fkMDo ++ mktemp + local LAST_ERR=/tmp/tmp.kJ2Pszb8Gq + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.34.61 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.DAHk9fkMDo + cat /tmp/tmp.kJ2Pszb8Gq + rm /tmp/tmp.DAHk9fkMDo /tmp/tmp.kJ2Pszb8Gq + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.XJtLFBxkzn/select-1.sql + for repl in '$(seq 1 3)' + desc 'check replica 3' + set +o xtrace ----------------------------------------------------------------------------------- check replica 3 ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.OznJu9TqwE ++ mktemp + local LAST_ERR=/tmp/tmp.MENs3xVCkK + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.OznJu9TqwE perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.MENs3xVCkK + rm /tmp/tmp.OznJu9TqwE /tmp/tmp.MENs3xVCkK + return 0 + sleep 30 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl3 + sleep 30 + check_replica some-name-repl3 some-name select-1 + local name=some-name-repl3 + local cluster=some-name + local sql_cmp=select-1 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2NsVUHAQ2C ++++ mktemp +++ local LAST_ERR=/tmp/tmp.0BQS5KmyKI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2NsVUHAQ2C +++ cat /tmp/tmp.0BQS5KmyKI +++ rm /tmp/tmp.2NsVUHAQ2C /tmp/tmp.0BQS5KmyKI +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.YRWFftqaPl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zafjEanWEw +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.YRWFftqaPl +++ cat /tmp/tmp.zafjEanWEw +++ rm /tmp/tmp.YRWFftqaPl /tmp/tmp.zafjEanWEw +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ozn5uk5WYL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ltB94Am5au +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Ozn5uk5WYL +++ cat /tmp/tmp.ltB94Am5au +++ rm /tmp/tmp.Ozn5uk5WYL /tmp/tmp.ltB94Am5au +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zE2cgWNauw +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Jj8tQZHzEF ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zE2cgWNauw ++++ cat /tmp/tmp.Jj8tQZHzEF ++++ rm /tmp/tmp.zE2cgWNauw /tmp/tmp.Jj8tQZHzEF ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MOVdNGsAHe ++++ mktemp +++ local LAST_ERR=/tmp/tmp.emZpNfEwgE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.MOVdNGsAHe +++ cat /tmp/tmp.emZpNfEwgE +++ rm /tmp/tmp.MOVdNGsAHe /tmp/tmp.emZpNfEwgE +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.YyqT6F2KRx +++ mktemp ++ local LAST_ERR=/tmp/tmp.1DAMH6XcNm ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.YyqT6F2KRx ++ cat /tmp/tmp.1DAMH6XcNm ++ rm /tmp/tmp.YyqT6F2KRx /tmp/tmp.1DAMH6XcNm ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.WtpPS9Nyh4 ++ mktemp + local LAST_ERR=/tmp/tmp.ZlTosJiMHN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.WtpPS9Nyh4 + cat /tmp/tmp.ZlTosJiMHN + rm /tmp/tmp.WtpPS9Nyh4 /tmp/tmp.ZlTosJiMHN + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.wx6XWflKT8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.MlQvdljVMN ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.wx6XWflKT8 ++ cat /tmp/tmp.MlQvdljVMN ++ rm /tmp/tmp.wx6XWflKT8 /tmp/tmp.MlQvdljVMN ++ return 0 + replica_pod_name=some-name-repl3-5fc65cdbb8-cjjjk ++ grep some-name-repl3-5fc65cdbb8-cjjjk /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl3-5fc65cdbb8-cjjjk | 10.87.32.32 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2OElD5bCrW +++ mktemp ++ local LAST_ERR=/tmp/tmp.QswzZRzLyw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2OElD5bCrW ++ cat /tmp/tmp.QswzZRzLyw ++ rm /tmp/tmp.2OElD5bCrW /tmp/tmp.QswzZRzLyw ++ return 0 + replica_pod_IP=10.87.32.32 + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.32.32 '' ' ' + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.32.32 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.32.32 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.32.32 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.srjb2LOwjY +++ mktemp ++ local LAST_ERR=/tmp/tmp.NJDdopXXrZ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.srjb2LOwjY ++ cat /tmp/tmp.NJDdopXXrZ ++ rm /tmp/tmp.srjb2LOwjY /tmp/tmp.NJDdopXXrZ ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.32.32 ' ++ mktemp + local LAST_OUT=/tmp/tmp.VKaPkFlmOD ++ mktemp + local LAST_ERR=/tmp/tmp.rfjBv6gFGL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.32.32 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.VKaPkFlmOD + cat /tmp/tmp.rfjBv6gFGL + rm /tmp/tmp.VKaPkFlmOD /tmp/tmp.rfjBv6gFGL + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.XJtLFBxkzn/select-1.sql + desc 'Restart the cluster' + set +o xtrace ----------------------------------------------------------------------------------- Restart the cluster ----------------------------------------------------------------------------------- + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]' ++ mktemp + local LAST_OUT=/tmp/tmp.xgwomgyfVB ++ mktemp + local LAST_ERR=/tmp/tmp.JTxbjaITOV + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xgwomgyfVB perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.JTxbjaITOV + rm /tmp/tmp.xgwomgyfVB /tmp/tmp.JTxbjaITOV + return 0 + sleep 30 ++ seq 1 3 + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl1........................................... + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl2 + for i in '$(seq 1 3)' + sleep 15 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl3......... + wait_deployment some-name + local name=some-name + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name....... + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.opEpKoDPhm ++ mktemp + local LAST_ERR=/tmp/tmp.LQ06bc8H0u + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.opEpKoDPhm perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.LQ06bc8H0u + rm /tmp/tmp.opEpKoDPhm /tmp/tmp.LQ06bc8H0u + return 0 + wait_for_delete pgreplica/some-name-repl3 + local res=pgreplica/some-name-repl3 + set +o xtrace pgreplica/some-name-repl3 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl3" not found + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.mKQTPf6lQ6 ++ mktemp + local LAST_ERR=/tmp/tmp.ZaadiisSkI + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.mKQTPf6lQ6 perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.ZaadiisSkI + rm /tmp/tmp.mKQTPf6lQ6 /tmp/tmp.ZaadiisSkI + return 0 + wait_for_delete pgreplica/some-name-repl2 + local res=pgreplica/some-name-repl2 + set +o xtrace pgreplica/some-name-repl2 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl2" not found + sleep 30 + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]' ++ mktemp + local LAST_OUT=/tmp/tmp.GH2ENS4sFH ++ mktemp + local LAST_ERR=/tmp/tmp.BZr3lhBaUv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.GH2ENS4sFH perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.BZr3lhBaUv + rm /tmp/tmp.GH2ENS4sFH /tmp/tmp.BZr3lhBaUv + return 0 + wait_for_delete pgreplica/some-name-repl1 + local res=pgreplica/some-name-repl1 + set +o xtrace pgreplica/some-name-repl1 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl1" not found + sleep 30 + kubectl_bin delete perconapgclusters --all ++ mktemp + local LAST_OUT=/tmp/tmp.gyBY6JmGIE ++ mktemp + local LAST_ERR=/tmp/tmp.l1zsDDRdpN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete perconapgclusters --all + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.gyBY6JmGIE perconapgcluster.pg.percona.com "some-name" deleted + cat /tmp/tmp.l1zsDDRdpN + rm /tmp/tmp.gyBY6JmGIE /tmp/tmp.l1zsDDRdpN + return 0 + wait_for_delete pgcluster/some-name + local res=pgcluster/some-name + set +o xtrace pgcluster/some-name - Error from server (NotFound): pgclusters.pg.percona.com "some-name" not found + wait_for_delete deployment/some-name + local res=deployment/some-name + set +o xtrace deployment/some-name - Error from server (NotFound): deployments.apps "some-name" not found + wait_for_delete deployment/some-name-backrest-shared-repo + local res=deployment/some-name-backrest-shared-repo + set +o xtrace deployment/some-name-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found + compare_kubectl pvc/some-name + local resource=pvc/some-name + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name.yml + local new_result=/tmp/tmp.XJtLFBxkzn/pvc_some-name.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml pvc/some-name + yq d - '**.creationTimestamp' ++ mktemp + yq d - metadata.managedFields + yq d - '**.namespace' + yq d - metadata.resourceVersion + yq d - metadata.selfLink + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + local LAST_OUT=/tmp/tmp.VYaDLc9jSg + yq d - '**.uid' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**.dataSource' + yq d - '**.clusterIPs' + yq d - '**.storageClassName' + yq d - '**.procMount' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.volumeName' + yq d - spec.nodeName + yq d - spec.volumeMode + yq d - '**.finalizers' + yq d - spec.manualSelector + yq d - '**.(name==NAMESPACE)' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - spec.podReplacementPolicy + yq d - '**."percona.com/*"' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**.healthCheckNodePort' + yq d - '**.enableServiceLinks' + yq d - '**.(name==PODNAME)' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - '**.nodePort' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.ipFamilyPolicy' + yq d - '**.(name==suffix)' + yq d - '**.controller-uid' + yq d - '**.allocateLoadBalancerNodePorts' ++ mktemp + yq d - status + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.imagePullSecrets' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.preemptionPolicy' + yq d - '**.internalTrafficPolicy' + yq d - '**.ipFamilies' + local LAST_ERR=/tmp/tmp.TznkzkeYjz + local exit_status=0 + local timeout=4 + set +o errexit + yq d - '**.clusterIP' + yq d - '**.image' + yq d - metadata.deletionTimestamp + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.creationTimestamp' ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml pvc/some-name + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.VYaDLc9jSg + cat /tmp/tmp.TznkzkeYjz + rm /tmp/tmp.VYaDLc9jSg /tmp/tmp.TznkzkeYjz + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/pvc_some-name.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name.yml /tmp/tmp.XJtLFBxkzn/pvc_some-name.yml + compare_kubectl pvc/some-name-repl1 + local resource=pvc/some-name-repl1 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl1.yml + local new_result=/tmp/tmp.XJtLFBxkzn/pvc_some-name-repl1.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml pvc/some-name-repl1 + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - metadata.selfLink ++ mktemp + yq d - metadata.resourceVersion + local LAST_OUT=/tmp/tmp.BKLyR8Jzsu + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**.procMount' + yq d - '**.storageClassName' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**.image' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**.clusterIP' + yq d - '**.volumeName' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**.dataSource' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - spec.volumeMode + yq d - '**.creationTimestamp' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.finalizers' + yq d - '**."volume.kubernetes.io/selected-node"' ++ mktemp + yq d - status + yq d - spec.nodeName + yq d - spec.podReplacementPolicy + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.(name==NAMESPACE)' + yq d - '**."percona.com/*"' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - spec.manualSelector + yq d - '**.ipFamilies' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.clusterIPs' + local LAST_ERR=/tmp/tmp.NAgJPjYN0c + local exit_status=0 + local timeout=4 + set +o errexit + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - '**.healthCheckNodePort' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - '**.nodePort' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - '**.imagePullSecrets' + yq d - '**.enableServiceLinks' + yq d - '**.(name==suffix)' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**.(name==PODNAME)' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.controller-uid' + yq d - '**.preemptionPolicy' ++ seq 0 2 + yq d - '**.ipFamilyPolicy' + yq d - metadata.deletionTimestamp + yq d - '**.internalTrafficPolicy' + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml pvc/some-name-repl1 + yq d - metadata.managedFields + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.BKLyR8Jzsu + cat /tmp/tmp.NAgJPjYN0c + rm /tmp/tmp.BKLyR8Jzsu /tmp/tmp.NAgJPjYN0c + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl1.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl1.yml /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl1.yml + compare_kubectl pvc/some-name-repl2 + local resource=pvc/some-name-repl2 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl2.yml + local new_result=/tmp/tmp.XJtLFBxkzn/pvc_some-name-repl2.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.namespace' + yq d - '**.uid' + yq d - '**.creationTimestamp' + yq d - metadata.resourceVersion + yq d - metadata.deletionTimestamp + yq d - '**.image' + yq d - '**.storageClassName' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - metadata.selfLink + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**.clusterIP' + yq d - spec.manualSelector + yq d - '**."percona.com/*"' + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - '**.(volumeMode==Filesystem).volumeMode' + yq d - status + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - spec.podReplacementPolicy + yq d - '**.(name==NAMESPACE)' + yq d - '**.nodePort' + yq d - '**.(name==suffix)' + yq d - '**.(name==COMMAND_OPTS)' + yq d - '**.imagePullSecrets' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**.enableServiceLinks' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - '**.internalTrafficPolicy' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.controller-uid' + yq d - '**.allocateLoadBalancerNodePorts' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.preemptionPolicy' + yq d - '**.healthCheckNodePort' + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.clusterIPs' + yq d - '**.dataSource' + yq d - '**.procMount' + yq d - spec.volumeMode + yq d - '**.(name==PODNAME)' + yq d - spec.nodeName + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**.volumeName' + yq d - '**.finalizers' + kubectl_bin get -o yaml pvc/some-name-repl2 ++ mktemp + local LAST_OUT=/tmp/tmp.8GJA1tp7KM ++ mktemp + local LAST_ERR=/tmp/tmp.qcnAfqtrSo + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml pvc/some-name-repl2 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8GJA1tp7KM + cat /tmp/tmp.qcnAfqtrSo + rm /tmp/tmp.8GJA1tp7KM /tmp/tmp.qcnAfqtrSo + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl2.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl2.yml /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl2.yml + compare_kubectl pvc/some-name-repl3 + local resource=pvc/some-name-repl3 + local postfix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl3.yml + local new_result=/tmp/tmp.XJtLFBxkzn/pvc_some-name-repl3.yml + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin get -o yaml pvc/some-name-repl3 + yq d - metadata.managedFields + yq d - '**.creationTimestamp' + yq d - '**.uid' + yq d - metadata.selfLink + yq d - '**.namespace' + yq d - '**.image' + yq d - metadata.deletionTimestamp + yq d - 'metadata.annotations."cloud.google.com/neg"' + yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"' + yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' + yq d - '**.clusterIP' + yq d - 'metadata.annotations."kubernetes.io/psp"' + yq d - '**.dataSource' + yq d - '**."batch.kubernetes.io/controller-uid"' + yq d - '**.clusterIPs' + yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"' + yq d - '**.creationTimestamp' + yq d - '**.procMount' + yq d - metadata.resourceVersion + yq d - '**.storageClassName' + yq d - '**.finalizers' + yq d - '**."kubernetes.io/pvc-protection"' + yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' + yq d - '**."volume.kubernetes.io/storage-provisioner"' + yq d - '**.volumeName' + yq d - spec.manualSelector + yq d - '**.nodePort' + yq d - '**.healthCheckNodePort' + yq d - '**.imagePullSecrets' + yq d - '**.enableServiceLinks' + yq d - '**.(name==NAMESPACE)' + yq d - 'metadata.ownerReferences.*.apiVersion' + yq d - '**.(name==COMMAND_OPTS)' + yq d - spec.volumeMode + yq d - spec.nodeName + yq d - '**.(name==PODNAME)' + yq d - '**.controller-uid' + yq d - 'spec.volumeClaimTemplates.*.apiVersion' ++ mktemp + yq d - spec.podReplacementPolicy + yq d - '**.preemptionPolicy' + yq d - 'spec.volumeClaimTemplates.*.kind' + yq d - '**."volume.kubernetes.io/selected-node"' + yq d - '**."percona.com/*"' + yq d - '**.(volumeMode==Filesystem).volumeMode' + local LAST_OUT=/tmp/tmp.KKdGdSYjp7 + yq d - status + yq d - '**.ipFamilies' + yq d - '**.ipFamilyPolicy' + yq d - '**.(name==suffix)' + yq d - '**.internalTrafficPolicy' + yq d - '**.allocateLoadBalancerNodePorts' ++ mktemp + local LAST_ERR=/tmp/tmp.qSFwQhOKAx + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get -o yaml pvc/some-name-repl3 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KKdGdSYjp7 + cat /tmp/tmp.qSFwQhOKAx + rm /tmp/tmp.KKdGdSYjp7 /tmp/tmp.qSFwQhOKAx + return 0 + case $(yq r ${new_result} 'kind') in ++ yq r /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl3.yml kind + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/pvc_some-name-repl3.yml /tmp/tmp.XJtLFBxkzn/pvc_some-name-repl3.yml + sleep 30 + create_user_secrets some-name + local cluster=some-name + local namespace=recreate-18368 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - metadata.name some-name-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n recreate-18368 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.FF72bjXRfi ++ mktemp + local LAST_ERR=/tmp/tmp.ChS862ZaJP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.FF72bjXRfi secret/some-name-users-secret configured + cat /tmp/tmp.ChS862ZaJP + rm /tmp/tmp.FF72bjXRfi /tmp/tmp.ChS862ZaJP + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n recreate-18368 apply -f - + yq w - metadata.name some-name-pmm-secret ++ mktemp + local LAST_OUT=/tmp/tmp.sUvKDXvxtU ++ mktemp + local LAST_ERR=/tmp/tmp.qCU8bnMVCv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.sUvKDXvxtU secret/some-name-pmm-secret configured + cat /tmp/tmp.qCU8bnMVCv + rm /tmp/tmp.sUvKDXvxtU /tmp/tmp.qCU8bnMVCv + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name + local backup=false + local restore_from=false + local restore_opts=false + local custom_config=false + local podAntiAffinity= + local schedule= + local schedule_type= + local tls_only= + local root_via_pgbouncer= + yq w - metadata.labels.crunchy-pgha-scope some-name + yq w - metadata.labels.deployment-name some-name + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - metadata.labels.pg-cluster some-name + yq w - metadata.name some-name + yq w - spec.clustername some-name + yq w - spec.keepData false + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - metadata.labels.name some-name + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.database some-name + yq w - spec.name some-name + yq w - spec.user some-name + yq w - spec.keepBackups false + yq w - spec.secretsName some-name-users-secret + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + case ${podAntiAffinity} in + case ${backup} in + [[ false != \f\a\l\s\e ]] + [[ false != \f\a\l\s\e ]] + [[ '' == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.XJtLFBxkzn/cr.yaml -n recreate-18368 ++ mktemp + local LAST_OUT=/tmp/tmp.KT6bOe9341 ++ mktemp + local LAST_ERR=/tmp/tmp.wJIlWL4OcB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl apply -f /tmp/tmp.XJtLFBxkzn/cr.yaml -n recreate-18368 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KT6bOe9341 perconapgcluster.pg.percona.com/some-name created + cat /tmp/tmp.wJIlWL4OcB + rm /tmp/tmp.KT6bOe9341 /tmp/tmp.wJIlWL4OcB + return 0 + wait_cluster_consistency some-name + cluster_name=some-name + wait_cluster_status some-name 'pgcluster Initialized' + local cluster_name=some-name + local 'target_status=pgcluster Initialized' + local target_namespace=recreate-18368 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LnSudPNgs0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.JWchybBDe4 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.LnSudPNgs0 ++ cat /tmp/tmp.JWchybBDe4 ++ rm /tmp/tmp.LnSudPNgs0 /tmp/tmp.JWchybBDe4 ++ return 0 + status_query='kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MDrDaiVrXG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fW78eYVClM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.MDrDaiVrXG +++ cat /tmp/tmp.fW78eYVClM +++ rm /tmp/tmp.MDrDaiVrXG /tmp/tmp.fW78eYVClM +++ return 0 + [[ '' == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TpTOk920Z1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.aeaiTNgqCO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TpTOk920Z1 +++ cat /tmp/tmp.aeaiTNgqCO +++ rm /tmp/tmp.TpTOk920Z1 /tmp/tmp.aeaiTNgqCO +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 2 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MQZxOpG9MJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PkYkrZOIVX +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.MQZxOpG9MJ +++ cat /tmp/tmp.PkYkrZOIVX +++ rm /tmp/tmp.MQZxOpG9MJ /tmp/tmp.PkYkrZOIVX +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 3 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.K22yT7vyZ1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EAkLEPtqsC +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.K22yT7vyZ1 +++ cat /tmp/tmp.EAkLEPtqsC +++ rm /tmp/tmp.K22yT7vyZ1 /tmp/tmp.EAkLEPtqsC +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 4 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.A7fIHQ9zhX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TZ9cryLczJ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.A7fIHQ9zhX +++ cat /tmp/tmp.TZ9cryLczJ +++ rm /tmp/tmp.A7fIHQ9zhX /tmp/tmp.TZ9cryLczJ +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 5 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5vbHIISGV6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.fAe6HW44HF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5vbHIISGV6 +++ cat /tmp/tmp.fAe6HW44HF +++ rm /tmp/tmp.5vbHIISGV6 /tmp/tmp.fAe6HW44HF +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 6 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JUBpslZ6xg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9TDMcJIPIl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JUBpslZ6xg +++ cat /tmp/tmp.9TDMcJIPIl +++ rm /tmp/tmp.JUBpslZ6xg /tmp/tmp.9TDMcJIPIl +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 7 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IfM2nRqTZP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.McDfjx12BT +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.IfM2nRqTZP +++ cat /tmp/tmp.McDfjx12BT +++ rm /tmp/tmp.IfM2nRqTZP /tmp/tmp.McDfjx12BT +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 20 ++ get_psql_user_pass some-name some-name ++ local user=some-name ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.pux3JzJ4og ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qGRZr03QWC +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.pux3JzJ4og +++ cat /tmp/tmp.qGRZr03QWC +++ rm /tmp/tmp.pux3JzJ4og /tmp/tmp.qGRZr03QWC +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yxSaGqgXrg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sDL5cF3rGI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.yxSaGqgXrg +++ cat /tmp/tmp.sDL5cF3rGI +++ rm /tmp/tmp.yxSaGqgXrg /tmp/tmp.sDL5cF3rGI +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ndehogdvaa ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uZrc2zUy3R +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Ndehogdvaa +++ cat /tmp/tmp.uZrc2zUy3R +++ rm /tmp/tmp.Ndehogdvaa /tmp/tmp.uZrc2zUy3R +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.YidvoLhJ40 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.HqFAsJVaP8 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.YidvoLhJ40 ++++ cat /tmp/tmp.HqFAsJVaP8 ++++ rm /tmp/tmp.YidvoLhJ40 /tmp/tmp.HqFAsJVaP8 ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8vxbAL16vd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FeQsJCEWAV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.8vxbAL16vd +++ cat /tmp/tmp.FeQsJCEWAV +++ rm /tmp/tmp.8vxbAL16vd /tmp/tmp.FeQsJCEWAV +++ return 0 ++ echo some-name_pass + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-18368 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name.recreate-18368 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-18368 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3LO4rYr5Yf +++ mktemp ++ local LAST_ERR=/tmp/tmp.bsvGS1HibF ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.3LO4rYr5Yf ++ cat /tmp/tmp.bsvGS1HibF ++ rm /tmp/tmp.3LO4rYr5Yf /tmp/tmp.bsvGS1HibF ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.8ALWou3Abj ++ mktemp + local LAST_ERR=/tmp/tmp.CfZY9dXin4 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.8ALWou3Abj + cat /tmp/tmp.CfZY9dXin4 + rm /tmp/tmp.8ALWou3Abj /tmp/tmp.CfZY9dXin4 + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.XJtLFBxkzn/select-1.sql ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mO1RZb0acy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TGRodoZUt0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mO1RZb0acy +++ cat /tmp/tmp.TGRodoZUt0 +++ rm /tmp/tmp.mO1RZb0acy /tmp/tmp.TGRodoZUt0 +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7OxFVPLvxc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.F4nnhfI1xV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7OxFVPLvxc +++ cat /tmp/tmp.F4nnhfI1xV +++ rm /tmp/tmp.7OxFVPLvxc /tmp/tmp.F4nnhfI1xV +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4M1WrQomRc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.LbHMOrk5gh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.4M1WrQomRc +++ cat /tmp/tmp.LbHMOrk5gh +++ rm /tmp/tmp.4M1WrQomRc /tmp/tmp.LbHMOrk5gh +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.kJSMwuKCxM +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.D6cN5PSmlG ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.kJSMwuKCxM ++++ cat /tmp/tmp.D6cN5PSmlG ++++ rm /tmp/tmp.kJSMwuKCxM /tmp/tmp.D6cN5PSmlG ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.M8NlGWKdAk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dtMYE2IRxQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.M8NlGWKdAk +++ cat /tmp/tmp.dtMYE2IRxQ +++ rm /tmp/tmp.M8NlGWKdAk /tmp/tmp.dtMYE2IRxQ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name.recreate-18368 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GvClanJdnb +++ mktemp ++ local LAST_ERR=/tmp/tmp.yLUrc4Z75j ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GvClanJdnb ++ cat /tmp/tmp.yLUrc4Z75j ++ rm /tmp/tmp.GvClanJdnb /tmp/tmp.yLUrc4Z75j ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.5NHmqf0m5l ++ mktemp + local LAST_ERR=/tmp/tmp.3fkIGyCpVH + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5NHmqf0m5l + cat /tmp/tmp.3fkIGyCpVH + rm /tmp/tmp.5NHmqf0m5l /tmp/tmp.3fkIGyCpVH + return 0 ++ seq 1 3 + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' ++ mktemp + local LAST_OUT=/tmp/tmp.tcL1asprte ++ mktemp + local LAST_ERR=/tmp/tmp.gk8W4VTFMz + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.tcL1asprte perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.gk8W4VTFMz + rm /tmp/tmp.tcL1asprte /tmp/tmp.gk8W4VTFMz + return 0 + sleep 25 + wait_deployment some-name-repl1 + local name=some-name-repl1 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl1 + sleep 35 + check_replica some-name-repl1 some-name select-2 + local name=some-name-repl1 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zoUp1SIHav ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hd8574LaKM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zoUp1SIHav +++ cat /tmp/tmp.hd8574LaKM +++ rm /tmp/tmp.zoUp1SIHav /tmp/tmp.hd8574LaKM +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.gXEkkSQRSd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Yge9teOBhX +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.gXEkkSQRSd +++ cat /tmp/tmp.Yge9teOBhX +++ rm /tmp/tmp.gXEkkSQRSd /tmp/tmp.Yge9teOBhX +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.fEFsq7uin3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.j8QLxqpoGv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.fEFsq7uin3 +++ cat /tmp/tmp.j8QLxqpoGv +++ rm /tmp/tmp.fEFsq7uin3 /tmp/tmp.j8QLxqpoGv +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.yNcTmQKvTd +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.asDsuRgGOF ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.yNcTmQKvTd ++++ cat /tmp/tmp.asDsuRgGOF ++++ rm /tmp/tmp.yNcTmQKvTd /tmp/tmp.asDsuRgGOF ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2B4pX3W2mg ++++ mktemp +++ local LAST_ERR=/tmp/tmp.AfoXOrVxJx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2B4pX3W2mg +++ cat /tmp/tmp.AfoXOrVxJx +++ rm /tmp/tmp.2B4pX3W2mg /tmp/tmp.AfoXOrVxJx +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.swcY7gL792 +++ mktemp ++ local LAST_ERR=/tmp/tmp.VF76wGYLno ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.swcY7gL792 ++ cat /tmp/tmp.VF76wGYLno ++ rm /tmp/tmp.swcY7gL792 /tmp/tmp.VF76wGYLno ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.Xi6W2QMBoH ++ mktemp + local LAST_ERR=/tmp/tmp.64xug6eHQR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Xi6W2QMBoH + cat /tmp/tmp.64xug6eHQR + rm /tmp/tmp.Xi6W2QMBoH /tmp/tmp.64xug6eHQR + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2asv7LbMB2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.iNszBIRyKy ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2asv7LbMB2 ++ cat /tmp/tmp.iNszBIRyKy ++ rm /tmp/tmp.2asv7LbMB2 /tmp/tmp.iNszBIRyKy ++ return 0 + replica_pod_name=some-name-repl1-5489f574fb-b5r2k ++ grep some-name-repl1-5489f574fb-b5r2k /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl1-5489f574fb-b5r2k | 10.87.33.56 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.2h0Bo0bmaU +++ mktemp ++ local LAST_ERR=/tmp/tmp.IVD9VxuWGj ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.2h0Bo0bmaU ++ cat /tmp/tmp.IVD9VxuWGj ++ rm /tmp/tmp.2h0Bo0bmaU /tmp/tmp.IVD9VxuWGj ++ return 0 + replica_pod_IP=10.87.33.56 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.33.56 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.33.56 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.33.56 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.33.56 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oFs5kisZ2X +++ mktemp ++ local LAST_ERR=/tmp/tmp.mkwvuUCeL2 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.oFs5kisZ2X ++ cat /tmp/tmp.mkwvuUCeL2 ++ rm /tmp/tmp.oFs5kisZ2X /tmp/tmp.mkwvuUCeL2 ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.33.56 ' ++ mktemp + local LAST_OUT=/tmp/tmp.TzHlfHIIPU ++ mktemp + local LAST_ERR=/tmp/tmp.mp7qHIq28y + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.33.56 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.TzHlfHIIPU + cat /tmp/tmp.mp7qHIq28y + rm /tmp/tmp.TzHlfHIIPU /tmp/tmp.mp7qHIq28y + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.XJtLFBxkzn/select-2.sql + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' ++ mktemp + local LAST_OUT=/tmp/tmp.cvXcQg7G9T ++ mktemp + local LAST_ERR=/tmp/tmp.xiBdmNBiMC + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.cvXcQg7G9T perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.xiBdmNBiMC + rm /tmp/tmp.cvXcQg7G9T /tmp/tmp.xiBdmNBiMC + return 0 + sleep 25 + wait_deployment some-name-repl2 + local name=some-name-repl2 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl2. + sleep 35 + check_replica some-name-repl2 some-name select-2 + local name=some-name-repl2 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XAZJxvotAx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.t86AqwmumA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XAZJxvotAx +++ cat /tmp/tmp.t86AqwmumA +++ rm /tmp/tmp.XAZJxvotAx /tmp/tmp.t86AqwmumA +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iAfLxL2ktb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.czmO7aUmCg +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iAfLxL2ktb +++ cat /tmp/tmp.czmO7aUmCg +++ rm /tmp/tmp.iAfLxL2ktb /tmp/tmp.czmO7aUmCg +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kSbQ2tDSbq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Cww2avM4cu +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.kSbQ2tDSbq +++ cat /tmp/tmp.Cww2avM4cu +++ rm /tmp/tmp.kSbQ2tDSbq /tmp/tmp.Cww2avM4cu +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.wWZ1XmcfTZ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.iEoD7HehuT ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.wWZ1XmcfTZ ++++ cat /tmp/tmp.iEoD7HehuT ++++ rm /tmp/tmp.wWZ1XmcfTZ /tmp/tmp.iEoD7HehuT ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.eaUAwK6XbJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.OW2C5qzBXi +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.eaUAwK6XbJ +++ cat /tmp/tmp.OW2C5qzBXi +++ rm /tmp/tmp.eaUAwK6XbJ /tmp/tmp.OW2C5qzBXi +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CnYxqfHoqH +++ mktemp ++ local LAST_ERR=/tmp/tmp.hep8FDDlv6 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.CnYxqfHoqH ++ cat /tmp/tmp.hep8FDDlv6 ++ rm /tmp/tmp.CnYxqfHoqH /tmp/tmp.hep8FDDlv6 ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.YhS0TqZl0K ++ mktemp + local LAST_ERR=/tmp/tmp.KVUAkWhyfj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.YhS0TqZl0K + cat /tmp/tmp.KVUAkWhyfj + rm /tmp/tmp.YhS0TqZl0K /tmp/tmp.KVUAkWhyfj + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.0TbyflqZR8 +++ mktemp ++ local LAST_ERR=/tmp/tmp.fKk7CjAIMJ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.0TbyflqZR8 ++ cat /tmp/tmp.fKk7CjAIMJ ++ rm /tmp/tmp.0TbyflqZR8 /tmp/tmp.fKk7CjAIMJ ++ return 0 + replica_pod_name=some-name-repl2-759b4f7fdd-hkdgj ++ grep some-name-repl2-759b4f7fdd-hkdgj /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl2-759b4f7fdd-hkdgj | 10.87.34.68 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FWtOb22k8a +++ mktemp ++ local LAST_ERR=/tmp/tmp.1r4NBnG7YZ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FWtOb22k8a ++ cat /tmp/tmp.1r4NBnG7YZ ++ rm /tmp/tmp.FWtOb22k8a /tmp/tmp.1r4NBnG7YZ ++ return 0 + replica_pod_IP=10.87.34.68 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.34.68 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.34.68 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.34.68 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.34.68 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EX6oDgVrCu +++ mktemp ++ local LAST_ERR=/tmp/tmp.zTMpekpV0c ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.EX6oDgVrCu ++ cat /tmp/tmp.zTMpekpV0c ++ rm /tmp/tmp.EX6oDgVrCu /tmp/tmp.zTMpekpV0c ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.34.68 ' ++ mktemp + local LAST_OUT=/tmp/tmp.XY8UI5LatD ++ mktemp + local LAST_ERR=/tmp/tmp.uaUilITNq3 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.34.68 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.XY8UI5LatD + cat /tmp/tmp.uaUilITNq3 + rm /tmp/tmp.XY8UI5LatD /tmp/tmp.uaUilITNq3 + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.XJtLFBxkzn/select-2.sql + for r in '$(seq 1 3)' + kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' ++ mktemp + local LAST_OUT=/tmp/tmp.7lJWfHkvrV ++ mktemp + local LAST_ERR=/tmp/tmp.uDsIqniGQm + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7lJWfHkvrV perconapgcluster.pg.percona.com/some-name patched + cat /tmp/tmp.uDsIqniGQm + rm /tmp/tmp.7lJWfHkvrV /tmp/tmp.uDsIqniGQm + return 0 + sleep 25 + wait_deployment some-name-repl3 + local name=some-name-repl3 + local target_namespace=recreate-18368 + sleep 10 + set +o xtrace some-name-repl3 + sleep 35 + check_replica some-name-repl3 some-name select-2 + local name=some-name-repl3 + local cluster=some-name + local sql_cmp=select-2 ++ get_psql_user_pass postgres some-name ++ local user=postgres ++ local cluster=some-name ++ local namespace=recreate-18368 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j3dmSRWm0a ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rAk4XsigGU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.j3dmSRWm0a +++ cat /tmp/tmp.rAk4XsigGU +++ rm /tmp/tmp.j3dmSRWm0a /tmp/tmp.rAk4XsigGU +++ return 0 ++ [[ xsome-name == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uZak7gWtdz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.k9LcmpwfdP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uZak7gWtdz +++ cat /tmp/tmp.k9LcmpwfdP +++ rm /tmp/tmp.uZak7gWtdz /tmp/tmp.k9LcmpwfdP +++ return 0 +++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.aqBIkaJBWJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Md2KB7YXg9 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.aqBIkaJBWJ +++ cat /tmp/tmp.Md2KB7YXg9 +++ rm /tmp/tmp.aqBIkaJBWJ /tmp/tmp.Md2KB7YXg9 +++ return 0 ++ [[ xsome-name-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zMMLRJO9uA +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ErBixvoOsp ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++++ kubectl -n recreate-18368 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zMMLRJO9uA ++++ cat /tmp/tmp.ErBixvoOsp ++++ rm /tmp/tmp.zMMLRJO9uA /tmp/tmp.ErBixvoOsp ++++ return 0 +++ kubectl_bin -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RiDEmejVH2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NhNxq4UcxK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox +++ kubectl -n recreate-18368 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RiDEmejVH2 +++ cat /tmp/tmp.NhNxq4UcxK +++ rm /tmp/tmp.RiDEmejVH2 /tmp/tmp.NhNxq4UcxK +++ return 0 ++ echo postgres_password + run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-18368 + local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication' + local uri=postgres:postgres_password@some-name.recreate-18368 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.58YS8Vh8Hp +++ mktemp ++ local LAST_ERR=/tmp/tmp.KwX7JL2vHR ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.58YS8Vh8Hp ++ cat /tmp/tmp.KwX7JL2vHR ++ rm /tmp/tmp.58YS8Vh8Hp /tmp/tmp.KwX7JL2vHR ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.13faepKJDC ++ mktemp + local LAST_ERR=/tmp/tmp.8xj0k3FpWg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-18368.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.13faepKJDC + cat /tmp/tmp.8xj0k3FpWg + rm /tmp/tmp.13faepKJDC /tmp/tmp.8xj0k3FpWg + return 0 ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.Ia5kGhBUlK +++ mktemp ++ local LAST_ERR=/tmp/tmp.f9F0UwGsTn ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.Ia5kGhBUlK ++ cat /tmp/tmp.f9F0UwGsTn ++ rm /tmp/tmp.Ia5kGhBUlK /tmp/tmp.f9F0UwGsTn ++ return 0 + replica_pod_name=some-name-repl3-5fc65cdbb8-6jxlm ++ grep some-name-repl3-5fc65cdbb8-6jxlm /tmp/tmp.XJtLFBxkzn/replicas.list ++ grep streaming + [[ -z primaryuser | some-name-repl3-5fc65cdbb8-6jxlm | 10.87.32.37 | streaming ]] ++ kubectl_bin -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ukGhvPjRAK +++ mktemp ++ local LAST_ERR=/tmp/tmp.IUVNkc2UxY ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ukGhvPjRAK ++ cat /tmp/tmp.IUVNkc2UxY ++ rm /tmp/tmp.ukGhvPjRAK /tmp/tmp.IUVNkc2UxY ++ return 0 + replica_pod_IP=10.87.32.37 + compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.32.37 '' ' ' + local command_id=select-2 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.32.37 + local postfix= + local 'uri_suffix= ' + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql + run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.87.32.37 postgres ' ' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=some-name:some-name_pass@10.87.32.37 + local driver=postgres + local 'suffix= ' ++ kubectl_bin -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hn2ipecsIR +++ mktemp ++ local LAST_ERR=/tmp/tmp.OjkWkA2Vm7 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl -n recreate-18368 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.hn2ipecsIR ++ cat /tmp/tmp.OjkWkA2Vm7 ++ rm /tmp/tmp.hn2ipecsIR /tmp/tmp.OjkWkA2Vm7 ++ return 0 + local client_container=pg-client-76df9df876-d7rc5 + kubectl_bin exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.32.37 ' ++ mktemp + local LAST_OUT=/tmp/tmp.5sCsBTs9TX ++ mktemp + local LAST_ERR=/tmp/tmp.MXdXUlZkA6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl exec pg-client-76df9df876-d7rc5 -n recreate-18368 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.87.32.37 ' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5sCsBTs9TX + cat /tmp/tmp.MXdXUlZkA6 + rm /tmp/tmp.5sCsBTs9TX /tmp/tmp.MXdXUlZkA6 + return 0 + '[' '!' -s /tmp/tmp.XJtLFBxkzn/select-2.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.XJtLFBxkzn/select-2.sql ++ kubectl_bin get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ALh9rKOeLe +++ mktemp ++ local LAST_ERR=/tmp/tmp.3kENOxvLiv ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ALh9rKOeLe ++ cat /tmp/tmp.3kENOxvLiv ++ rm /tmp/tmp.ALh9rKOeLe /tmp/tmp.3kENOxvLiv ++ return 0 + master_container=some-name-5d6f899994-m7gsk + kubectl_bin get configmap/some-name-pgha-config -o yaml + yq r - 'data[some-name-dcs-config]' + yq w - postgresql.parameters.jit false ++ mktemp + local LAST_OUT=/tmp/tmp.pORwsOfwZI ++ mktemp + local LAST_ERR=/tmp/tmp.cAYfH0wW2V + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get configmap/some-name-pgha-config -o yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.pORwsOfwZI + cat /tmp/tmp.cAYfH0wW2V + rm /tmp/tmp.pORwsOfwZI /tmp/tmp.cAYfH0wW2V + return 0 + kubectl_bin apply -f - + kubectl_bin get configmap/some-name-pgha-config -o yaml ++ mktemp ++ mktemp ++ cat /tmp/tmp.XJtLFBxkzn/some-name-pgha-config.patch + local LAST_OUT=/tmp/tmp.QLjVskf5WB + local LAST_OUT=/tmp/tmp.19bVG8NaS8 + yq w - 'data[some-name-dcs-config]' 'postgresql: parameters: jit: false unix_socket_directories: /tmp wal_level: logical archive_mode: true archive_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-push "%p" archive_timeout: 60 log_directory: pg_log shared_buffers: 128MB temp_buffers: 8MB log_min_duration_statement: 60000 log_statement: none work_mem: 4MB max_wal_senders: 6 shared_preload_libraries: pg_stat_monitor,pgaudit ssl: "on" ssl_cert_file: /pgconf/tls/tls.crt ssl_key_file: /pgconf/tls/tls.key ssl_ca_file: /pgconf/tls/ca.crt use_slots: false recovery_conf: restore_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-get %f "%p" use_pg_rewind: true' ++ mktemp + local LAST_ERR=/tmp/tmp.QY38ziliPV + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 ++ mktemp + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get configmap/some-name-pgha-config -o yaml + local LAST_ERR=/tmp/tmp.Mq0hi6rBNW + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.QLjVskf5WB + cat /tmp/tmp.QY38ziliPV + rm /tmp/tmp.QLjVskf5WB /tmp/tmp.QY38ziliPV + return 0 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.19bVG8NaS8 configmap/some-name-pgha-config configured + cat /tmp/tmp.Mq0hi6rBNW Warning: resource configmaps/some-name-pgha-config is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically. + rm /tmp/tmp.19bVG8NaS8 /tmp/tmp.Mq0hi6rBNW + return 0 ++ kubectl_bin get configmap/some-name-pgha-config -o yaml ++ yq r - 'data[some-name-dcs-config]' ++ yq r - postgresql.parameters.jit +++ mktemp ++ local LAST_OUT=/tmp/tmp.nGxJ5NsHcu +++ mktemp ++ local LAST_ERR=/tmp/tmp.eejukNNk3I ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get configmap/some-name-pgha-config -o yaml ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.nGxJ5NsHcu ++ cat /tmp/tmp.eejukNNk3I ++ rm /tmp/tmp.nGxJ5NsHcu /tmp/tmp.eejukNNk3I ++ return 0 + CONF_MAP_JIT=false ++ kubectl_bin exec some-name-5d6f899994-m7gsk -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" ' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9jEFxxhwN1 +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZYAenooR09 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl exec some-name-5d6f899994-m7gsk -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" ' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9jEFxxhwN1 ++ cat /tmp/tmp.ZYAenooR09 ++ rm /tmp/tmp.9jEFxxhwN1 /tmp/tmp.ZYAenooR09 ++ return 0 + PATRONI_JIT=false + [[ false != false ]] + destroy recreate-18368 + grep -v level=info + grep -v level=debug + grep -v 'Getting tasks for pod' + grep -v 'the object has been modified' + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + grep -v 'Getting pods from source' + grep -v 'get backup status: Job.batch' + tee /tmp/tmp.XJtLFBxkzn/operator.log + sort -u ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.GZ23wegjvU +++ mktemp ++ local LAST_ERR=/tmp/tmp.uw1yDPpL1H ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.GZ23wegjvU ++ cat /tmp/tmp.uw1yDPpL1H ++ rm /tmp/tmp.GZ23wegjvU /tmp/tmp.uw1yDPpL1H ++ return 0 + kubectl_bin logs postgres-operator-c6cccc8b9-gr4rq -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.9TIUNwe4pr ++ mktemp + local LAST_ERR=/tmp/tmp.mlsuMwDnXH + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl logs postgres-operator-c6cccc8b9-gr4rq -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9TIUNwe4pr + cat /tmp/tmp.mlsuMwDnXH + rm /tmp/tmp.9TIUNwe4pr /tmp/tmp.mlsuMwDnXH + return 0 ] { } }, ] } }, { } }, ] { } }, { } ] } { } }, ] } ] { } }, }, { ] { } }, ] ] ], { } }, }, { }, { },{ }] }], ] { } }, , ], { } }, }], { } }, }], } ], } }, } }, } }, { } }, }, 2024/05/24 19:53:56 INF 1 (localhost:4150) connecting to nsqd 2024/05/24 19:54:45 INF 2 (localhost:4150) connecting to nsqd 2024/05/24 19:54:46 INF 3 (localhost:4150) connecting to nsqd 2024/05/24 19:54:53 INF 4 (localhost:4150) connecting to nsqd 2024/05/24 19:55:21 INF 5 (localhost:4150) connecting to nsqd 2024/05/24 19:55:21 INF 6 (localhost:4150) connecting to nsqd 2024/05/24 19:55:21 INF 7 (localhost:4150) connecting to nsqd 2024/05/24 19:56:25 INF 8 (localhost:4150) connecting to nsqd 2024/05/24 19:56:44 INF 9 (localhost:4150) connecting to nsqd 2024/05/24 19:57:59 INF 10 (localhost:4150) connecting to nsqd 2024/05/24 19:59:21 INF 11 (localhost:4150) connecting to nsqd 2024/05/24 20:00:43 INF 12 (localhost:4150) connecting to nsqd 2024/05/24 20:07:07 INF 13 (localhost:4150) connecting to nsqd 2024/05/24 20:07:57 INF 14 (localhost:4150) connecting to nsqd 2024/05/24 20:08:47 INF 15 (localhost:4150) connecting to nsqd 2024/05/24 20:09:31 INF 16 (localhost:4150) connecting to nsqd 2024/05/24 20:09:47 INF 17 (localhost:4150) connecting to nsqd 2024/05/24 20:09:52 INF 18 (localhost:4150) connecting to nsqd 2024/05/24 20:10:48 INF 19 (localhost:4150) connecting to nsqd 2024/05/24 20:11:32 INF 20 (localhost:4150) connecting to nsqd 2024/05/24 20:11:33 INF 21 (localhost:4150) connecting to nsqd 2024/05/24 20:11:40 INF 22 (localhost:4150) connecting to nsqd 2024/05/24 20:11:54 INF 23 (localhost:4150) connecting to nsqd 2024/05/24 20:11:55 INF 24 (localhost:4150) connecting to nsqd 2024/05/24 20:11:55 INF 25 (localhost:4150) connecting to nsqd 2024/05/24 20:12:30 INF 26 (localhost:4150) connecting to nsqd 2024/05/24 20:13:50 INF 27 (localhost:4150) connecting to nsqd 2024/05/24 20:15:11 INF 28 (localhost:4150) connecting to nsqd "accessModes": [ "affinity": "affinity": { "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "apiVersion": "v1", "apiVersion": "v1", "backrest-command": "backup", "backrest-command": "stanza-create", "claimName": "some-name-pgbr-repo" "command": [ "configMap": { "configMap": { "configMapKeyRef": { "containerName": "database", "containerPort": 2022, "containerPort": 5432, "containerPort": 8009, "containers": [ "containers": [{ "cpu": "1", "cpu": "200m", "cpu": "500m", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "defaultMode": 288 "defaultMode": 288, "defaultMode": 288, "defaultMode": 420, "deployment-name": "some-name" "deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.7.0","name": "some-name","pg-cluster": "some-name" "deployment-name": "some-name-repl1" "deployment-name": "some-name-repl2" "deployment-name": "some-name-repl2","pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name" "deployment-name": "some-name-repl3" "deployment-name": "some-name-repl3","pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name" "divisor": "1m", "dnsPolicy": "ClusterFirst" "downwardAPI": { "emptyDir": { "emptyDir": { "emptyDir": { "emptyDir": { "env": [ "env": [{ "exec": { "fieldPath": "metadata.annotations" "fieldPath": "metadata.labels" "fieldPath": "metadata.labels['crunchy-pgha-scope']" "fieldPath": "metadata.namespace" "fieldRef": { "fieldRef": { "fsGroup": 2, I0524 19:53:20.658400 1 shared_informer.go:240] Waiting for caches to sync for namespace I0524 19:53:20.758672 1 shared_informer.go:247] Caches are synced for namespace I0524 19:53:20.837189 1 shared_informer.go:240] Waiting for caches to sync for recreate-18368 I0524 19:53:21.038187 1 shared_informer.go:247] Caches are synced for recreate-18368 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha", "image": "perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-rmdata", "imagePullPolicy": "IfNotPresent" "imagePullPolicy": "IfNotPresent", "initialDelaySeconds": 15 "initialDelaySeconds": 30, "items": [ "items": [ "items": [ "key": "config", "key": "config", "key": "crunchy-pgbouncer", "key": "init" "key": "password" "key": "pg-cluster", "key": "pgo-backrest", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "tls.crt", "key": "tls.key", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "PersistentVolumeClaim", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "livenessProbe": { "matchExpressions": [ "matchLabels": { "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory" "medium": "Memory", "medium": "Memory", "medium": "Memory", "memory": "128Mi" "memory": "256Mi" "memory": "48Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/backrestrepo", "mountPath": "/dev/shm", "mountPath": "/etc/pgbackrest" "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/podinfo", "mountPath": "/etc/ssh", "mountPath": "/etc/ssh", "mountPath": "/home/pgbackrest" "mountPath": "/pgconf", "mountPath": "/pgconf/", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/pgconf/tls-replication", "mountPath": "/pgdata", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp" "name": "backrest", "name": "backrest-backup-some-name", "name": "backrestrepo", "name": "backrestrepo", "name": "BACKREST_SKIP_CREATE_STANZA", "name": "cluster1-replication-ssl-keypair", "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "COMMAND", "name": "COMMAND_OPTS", "name": "database", "name": "dshm" "name": "dshm", "name": "ENABLE_SSHD", "name": "IS_BACKUP", "name": "IS_REPLICA", "name": "LD_PRELOAD", "name": "LD_PRELOAD", "name": "MODE", "name": "NAMESPACE", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_PASSWD", "name": "NSS_WRAPPER_PASSWD", "name": "patroni" "name": "PATRONI_KUBERNETES_LABELS", "name": "PATRONI_KUBERNETES_NAMESPACE", "name": "PATRONI_KUBERNETES_SCOPE_LABEL", "name": "PATRONI_LOG_LEVEL", "name": "PATRONI_POSTGRESQL_DATA_DIR", "name": "PATRONI_SCOPE", "name": "pgbackrest" "name": "pgbackrest-conf", "name": "pgbackrest-conf", "name": "pgbackrest-config" "name": "pgbackrest-config", "name": "PGBACKREST_DB_HOST", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_DB_PATH", "name": "pgbackrest-home", "name": "pgbackrest-home", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_REPO1_HOST", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "PG_CLUSTER", "name": "pgconf-volume" "name": "pgconf-volume", "name": "pgdata", "name": "pgdata", "name": "PGHA_DATABASE", "name": "PGHA_INIT", "name": "PGHA_PASSWORD_TYPE", "name": "PGHA_PGBACKREST", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PG_PORT", "name": "PGHA_REPLICA_REINIT_ON_START_FAIL", "name": "PGHA_SCOPE", "name": "PGHA_STANDBY", "name": "PGHA_SYNC_REPLICATION", "name": "PGHA_TLS_ENABLED", "name": "PGHA_TLS_ONLY", "name": "PGHA_USER", "name": "PGHOST", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "podinfo" "name": "podinfo", "name": "PODNAME", "name": "postgres" "name": "primary-volume" "name": "primary-volume", "name": "REMOVE_BACKUP", "name": "REMOVE_DATA", "name": "REPLICA_NAME", "name": "report", "name": "rmdata", "name": "root-volume" "name": "root-volume", "name": "some-name", "name": "some-name", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo" "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-backrest-shared-repo", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer", "name": "some-name-pgbouncer-cm" "name": "some-name-pgbouncer-secret", "name": "some-name-pgbouncer-secret", "name": "some-name-pgbr-repo", "name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.7.0" "name": "some-name-pgha-config", "name": "some-name-pgha-config", "name": "some-name-repl1", "name": "some-name-repl1", "name": "some-name-repl2", "name": "some-name-repl2", "name": "some-name-repl3", "name": "some-name-repl3", "name": "some-name-rmdata-ouqk", "name": "some-name-rmdata-ouqk", "name": "some-name-stanza-create", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "sshd", "name": "sshd", "name": "sshd", "name": "sshd", "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tls-replication" "name": "tls-replication", "name": "tls-server" "name": "tls-server", "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "name": "tmp", "name": "user-volume" "name": "user-volume", "nodePort": 0 "operator": "Exists" "operator": "In", "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh" "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh" "optional": true "path": "annotations" "path": "cpu_limit", "path": "cpu_request", "path": "labels" "path": "mem_limit", "path": "mem_request", "path": "ssh_config" "path": "ssh_config" "path": "tls-replication.crt" "path": "tls-replication.key" "periodSeconds": 15, "persistentVolumeClaim": { "persistentVolumeClaim":{"claimName":"some-name"} "persistentVolumeClaim":{"claimName":"some-name-repl1"} "persistentVolumeClaim":{"claimName":"some-name-repl2"} "persistentVolumeClaim":{"claimName":"some-name-repl3"} "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name" "pg-cluster": "some-name", "pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.7.0","name": "some-name" "pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.7.0","name": "some-name-replica" "pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.7.0","name": "some-name-replica" "pg-cluster": "some-name","deployment-name": "some-name-repl3","pgo-version": "1.7.0","name": "some-name-replica" "pgo-backrest-job": "true", "pgo-backrest-repo": "true" "pgo-backrest-repo": "true", "pgo-backrest-repo": "true" "pgo-backrest": "true", "pgo-pg-database": "true", "pgo-pg-database": "true", "pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.7.0","name": "some-name","pg-cluster": "some-name","deployment-name": "some-name" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3" "pgo-version": "1.7.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3" "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "pgrmdata": "true", "pgrmdata": "true", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "port": 2022, "ports": [{ "ports": [{ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "projected": { "projected": { "projected": { "sources": [] } "protocol": "TCP", "protocol": "TCP", "readinessProbe": { "readOnly": false "readOnly": false "readOnly": false "readOnlyRootFilesystem": true "readOnly": true "readOnly": true "readOnly": true "ReadWriteOnce" "replicas": 0, "replicas": 1, "replicas": 3, "requests": { "requests": { "resourceFieldRef": { "resource": "limits.cpu" "resource": "limits.memory" "resource": "requests.cpu" "resource": "requests.memory" "resources": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secret": { "secret": { "secret": { "secret": { "secretKeyRef": { "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-backrest-repo-config" "secretName": "some-name-backrest-repo-config", "secretName": "some-name-postgres-secret" "secretName": "some-name-primaryuser-secret" "secretName": "some-name-some-name-secret" "securityContext": { "securityContext": { "securityContext": {}, "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]}, "securityContext": {"runAsNonRoot":true,"fsGroup":26}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "serviceAccountName": "pgo-pg", "serviceAccountName": "pgo-target", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-backrest-shared-repo", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "service-name": "some-name-pgbouncer", "sessionAffinity": "None" "sizeLimit": "128Ki" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "sizeLimit": "2Mi" "sizeLimit": "64Mi" "sizeLimit": "64Mi" "some-name" "sources": [ "sources": [ "sources": [ "spec": { "spec": { "spec": { "storage": "1Gi" "strategy": { "targetPort": 2022, "template": { time="2024-05-24T19:53:20Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.7.0 time="2024-05-24T19:54:01Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T19:54:06Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T19:54:45Z" level=error msg="Error completing workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T19:54:45Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T19:55:21Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T19:55:21Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T19:55:22Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T19:59:21Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T19:59:22Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:00:43Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:03:25Z" level=warning msg="readiness timeout reached for deployment \"some-name-repl1\"" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.RollingUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:116" version=1.7.0 time="2024-05-24T20:03:29Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:33Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:37Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:41Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:45Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:49Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:53Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:03:57Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:01Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:05Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:09Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:13Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:17Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:21Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:25Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:04:25Z" level=warning msg="readiness timeout reached for start up of cluster \"some-name\" instance \"some-name-repl1\"" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.RollingUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:122" version=1.7.0 time="2024-05-24T20:05:21Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.7.0 time="2024-05-24T20:05:21Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:05:21Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.7.0 time="2024-05-24T20:06:09Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T20:06:09Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.7.0 time="2024-05-24T20:09:31Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:32Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:33Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:34Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:35Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:37Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:41Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:44Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:46Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:47Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T20:09:52Z" level=error msg="jobs.batch \"some-name-rmdata-ouqk\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.7.0 time="2024-05-24T20:10:52Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T20:10:57Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T20:11:32Z" level=error msg="Error completing workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T20:11:32Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T20:11:55Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:11:55Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:11:56Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:13:50Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T20:15:11Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 "timeoutSeconds": 10 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "5432" "value": "5432" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "/backrestrepo/some-name-backrest-shared-repo" "value": "backup" "value": "crunchy-pgha-scope" "value": "db" "value": "db" "value": " --db-host=10.87.32.29 --db-path=/pgdata/some-name" "value": " --db-host=10.87.32.35 --db-path=/pgdata/some-name" "value": "false" "value": "false" "valueFrom": { "value": "INFO" "value": "pgbackrest" "value": "pgbackrest-repo" "value": "/pgdata/some-name" "value": "/pgdata/some-name" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl1" "value": "/pgdata/some-name-repl2" "value": "/pgdata/some-name-repl2" "value": "/pgdata/some-name-repl3" "value": "/pgdata/some-name-repl3" "value": "posix" "value": "posix" "value": "postgres" "values": [ "value": "some-name" "value": "some-name-backrest-shared-repo" "value": "some-name-backrest-shared-repo-6f8f947dbc-92xvc" "value": "some-name-backrest-shared-repo-6f8f947dbc-vp287" "value": "stanza-create" "value": "/tmp" "value": "/tmp" "value": "/tmp/nss_wrapper/pgbackrest-repo/group" "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd" "value": "/tmp/nss_wrapper/postgres/group" "value": "/tmp/nss_wrapper/postgres/passwd" "value": "true" "value": "true" "value": "--type=full --db-host=10.87.32.29 --db-path=/pgdata/some-name" "value": "--type=full --db-host=10.87.32.35 --db-path=/pgdata/some-name" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/lib64/libnss_wrapper.so" "value": "{vendor: \"crunchydata\"}" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumeMounts": [{ "volumes": [ "volumes": [{ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.04DWNIbaHS ++ mktemp + local LAST_ERR=/tmp/tmp.GTy7fUD4BA + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete -f /tmp/tmp.XJtLFBxkzn/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.04DWNIbaHS + cat /tmp/tmp.GTy7fUD4BA Error from server (NotFound): error when deleting "/tmp/tmp.XJtLFBxkzn/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.XJtLFBxkzn/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.XJtLFBxkzn/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.XJtLFBxkzn/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.XJtLFBxkzn/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.04DWNIbaHS /tmp/tmp.GTy7fUD4BA + return 1 + true + yq w -d4 /tmp/tmp.XJtLFBxkzn/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.XJtLFBxkzn/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.ybzSXFB6f0 ++ mktemp + local LAST_ERR=/tmp/tmp.xAu7fGKCcj + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl apply -f /tmp/tmp.XJtLFBxkzn/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.ybzSXFB6f0 serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.xAu7fGKCcj + rm /tmp/tmp.ybzSXFB6f0 /tmp/tmp.xAu7fGKCcj + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=recreate-18368 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n recreate-18368 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.XJtLFBxkzn/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.51BgpmNHYe ++ mktemp + local LAST_ERR=/tmp/tmp.oFSRpuUbPR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete -f /tmp/tmp.XJtLFBxkzn/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.51BgpmNHYe serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.oFSRpuUbPR + rm /tmp/tmp.51BgpmNHYe /tmp/tmp.oFSRpuUbPR + return 0 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' + awk '{print$1}' + xargs kubectl delete ns --grace-period=0 --force=true ++ mktemp + local LAST_OUT=/tmp/tmp.2v9yqk4fiT ++ mktemp + local LAST_ERR=/tmp/tmp.jlqarmziAC + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2v9yqk4fiT + cat /tmp/tmp.jlqarmziAC + rm /tmp/tmp.2v9yqk4fiT /tmp/tmp.jlqarmziAC + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "recreate-18368" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.RxB8r3Ltsx ++ mktemp + local LAST_ERR=/tmp/tmp.VV99X70CLa + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.RxB8r3Ltsx + cat /tmp/tmp.VV99X70CLa Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.RxB8r3Ltsx /tmp/tmp.VV99X70CLa + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.pxHnX4v1s4 ++ mktemp + local LAST_ERR=/tmp/tmp.zpOkDNiRcN + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-sandbox + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.pxHnX4v1s4 + cat /tmp/tmp.zpOkDNiRcN Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.pxHnX4v1s4 /tmp/tmp.zpOkDNiRcN + return 1 + true + rm -rf /tmp/tmp.XJtLFBxkzn