++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/recreate.log'
Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/recreate.log
++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP=
++ oc get projects
error: the server doesn't have a resource type "projects"
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+++ helm version -c
+++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/'
++ HELM_VERSION=v3.14.4
++ '[' v3 == v2 ']'
+ create_namespace recreate-1535
+ local namespace=recreate-1535
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ cleanup_rbac
++ mktemp
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ xargs kubectl delete ns
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.yVj4LChvda
++ mktemp
+ local LAST_OUT=/tmp/tmp.P5IJyLRyLd
+ local LAST_ERR=/tmp/tmp.ejvNcBN932
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.6J7Hooanob
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.yVj4LChvda
+ cat /tmp/tmp.ejvNcBN932
+ rm /tmp/tmp.yVj4LChvda /tmp/tmp.ejvNcBN932
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.P5IJyLRyLd
+ cat /tmp/tmp.6J7Hooanob
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.P5IJyLRyLd /tmp/tmp.6J7Hooanob
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
++ mktemp
+ local LAST_OUT=/tmp/tmp.eJm0uWTht7
++ mktemp
+ local LAST_ERR=/tmp/tmp.bzkqV14CUh
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.eJm0uWTht7
+ cat /tmp/tmp.bzkqV14CUh
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.eJm0uWTht7 /tmp/tmp.bzkqV14CUh
+ return 1
+ true
+ kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth
Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found
+ true
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin delete namespace recreate-1535
++ mktemp
+ local LAST_OUT=/tmp/tmp.a3LtsjcXff
++ mktemp
+ local LAST_ERR=/tmp/tmp.I5xlkgfCo6
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace recreate-1535
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace recreate-1535
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace recreate-1535
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.a3LtsjcXff
+ cat /tmp/tmp.I5xlkgfCo6
Error from server (NotFound): namespaces "recreate-1535" not found
+ rm /tmp/tmp.a3LtsjcXff /tmp/tmp.I5xlkgfCo6
+ return 1
+ :
+ wait_for_delete namespace/recreate-1535
+ local res=namespace/recreate-1535
+ set +o xtrace
namespace/recreate-1535 - Error from server (NotFound): namespaces "recreate-1535" not found
+ kubectl_bin create namespace recreate-1535
++ mktemp
+ local LAST_OUT=/tmp/tmp.mVjdUHV6LK
++ mktemp
+ local LAST_ERR=/tmp/tmp.nv0IHmEJm5
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl create namespace recreate-1535
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.mVjdUHV6LK
namespace/recreate-1535 created
+ cat /tmp/tmp.nv0IHmEJm5
+ rm /tmp/tmp.mVjdUHV6LK /tmp/tmp.nv0IHmEJm5
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Vf5EjxZVdY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mJPs870RVB
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl config current-context
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Vf5EjxZVdY
++ cat /tmp/tmp.mJPs870RVB
++ rm /tmp/tmp.Vf5EjxZVdY /tmp/tmp.mJPs870RVB
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=recreate-1535
++ mktemp
+ local LAST_OUT=/tmp/tmp.X5Xlw6nTBU
++ mktemp
+ local LAST_ERR=/tmp/tmp.DsA6Snoy0I
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=recreate-1535
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.X5Xlw6nTBU
Context "gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox" modified.
+ cat /tmp/tmp.DsA6Snoy0I
+ rm /tmp/tmp.X5Xlw6nTBU /tmp/tmp.DsA6Snoy0I
+ return 0
+ deploy_operator
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local action=install
+ local namespace=recreate-1535
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled install recreate-1535 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local operator_action=install
+ local namespace=recreate-1535
+ local dont_send_telemetry=false
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "recreate-1535"#g'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "recreate-1535"#g'
+ cat /dev/fd/63 -
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
++ echo 'crunchy_debug: "true"'
++ cut -d: -f2
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-721-a67b3475b"#g'
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
++ cut -d: -f1
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml metadata.namespace recreate-1535
+ yq w -d3 - 'subjects[0].namespace' recreate-1535
+ yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install
++ cat /tmp/tmp.u8o2J7yvV1/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "recreate-1535"
namespace_mode: "disabled"
pgo_image_prefix: "perconalab/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "PR-721-a67b3475b"
pgo_operator_namespace: "recreate-1535"'
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.u8o2J7yvV1/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.PIn370jFLF
++ mktemp
+ local LAST_ERR=/tmp/tmp.aIFouN3AnJ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.u8o2J7yvV1/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.PIn370jFLF
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.aIFouN3AnJ
+ rm /tmp/tmp.PIn370jFLF /tmp/tmp.aIFouN3AnJ
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=recreate-1535
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 32 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 33 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 34 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 35 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.174wSS33jv
++ mktemp
+ local LAST_ERR=/tmp/tmp.blD4zwqnxR
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.174wSS33jv
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.blD4zwqnxR
+ rm /tmp/tmp.174wSS33jv /tmp/tmp.blD4zwqnxR
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.j3TtiWj3YT
+++ mktemp
++ local LAST_ERR=/tmp/tmp.XbctsJRvjq
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.j3TtiWj3YT
++ cat /tmp/tmp.XbctsJRvjq
++ rm /tmp/tmp.j3TtiWj3YT /tmp/tmp.XbctsJRvjq
++ return 0
+ wait_pod postgres-operator-56696f7849-zfjp8
+ local pod=postgres-operator-56696f7849-zfjp8
+ local ns=
+ set +o xtrace
postgres-operator-56696f7849-zfjp8true
+ [[ -n '' ]]
+ cluster=some-name
+ spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local backup=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ local namespace=recreate-1535
+ local root_via_pgbouncer=
+ desc 'create fresh PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create fresh PG cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name recreate-1535
+ local cluster=some-name
+ local namespace=recreate-1535
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - stringData.some-name some-name_pass
+ yq w - metadata.name some-name-users-secret
+ kubectl_bin -n recreate-1535 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.NzFb5OgNZ0
++ mktemp
+ local LAST_ERR=/tmp/tmp.pstG7hsUHd
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.NzFb5OgNZ0
secret/some-name-users-secret created
+ cat /tmp/tmp.pstG7hsUHd
+ rm /tmp/tmp.NzFb5OgNZ0 /tmp/tmp.pstG7hsUHd
+ return 0
+ yq w - metadata.name some-name-pmm-secret
+ kubectl_bin -n recreate-1535 apply -f -
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.rTBByFNTeD
++ mktemp
+ local LAST_ERR=/tmp/tmp.aPPaPc6aHq
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.rTBByFNTeD
secret/some-name-pmm-secret created
+ cat /tmp/tmp.aPPaPc6aHq
+ rm /tmp/tmp.rTBByFNTeD /tmp/tmp.aPPaPc6aHq
+ return 0
+ kubectl_bin -n recreate-1535 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.GEF8eXukNq
++ mktemp
+ local LAST_ERR=/tmp/tmp.cyox1YH5ON
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.GEF8eXukNq
deployment.apps/pg-client created
+ cat /tmp/tmp.cyox1YH5ON
+ rm /tmp/tmp.GEF8eXukNq /tmp/tmp.cyox1YH5ON
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name false false false false false false false false
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.labels.deployment-name some-name
+ yq w - metadata.labels.name some-name
+ yq w - spec.database some-name
+ yq w - spec.name some-name
+ yq w - spec.keepData false
+ yq w - spec.keepBackups false
+ yq w - spec.clustername some-name
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - metadata.name some-name
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.user some-name
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.secretsName some-name-users-secret
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.u8o2J7yvV1/cr.yaml -n recreate-1535
++ mktemp
+ local LAST_OUT=/tmp/tmp.K88PrdILI8
++ mktemp
+ local LAST_ERR=/tmp/tmp.f1DaCmm5Fr
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.u8o2J7yvV1/cr.yaml -n recreate-1535
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.K88PrdILI8
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.f1DaCmm5Fr
+ rm /tmp/tmp.K88PrdILI8 /tmp/tmp.f1DaCmm5Fr
+ return 0
+ wait_deployment some-name-backrest-shared-repo
+ local name=some-name-backrest-shared-repo
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-backrest-shared-repo..
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name..........
+ [[ false == \g\c\s\+* ]]
+ wait_job_completion some-name-stanza-create false
+ local job=some-name-stanza-create
+ local mandatory=false
+ local ns=recreate-1535
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
some-name-stanza-create+ retry=0
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name
+ local job=backrest-backup-some-name
+ local mandatory=true
+ local ns=recreate-1535
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
backrest-backup-some-name+ retry=0
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=recreate-1535
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tzll9r9yUX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.dSTRuEH9vd
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.tzll9r9yUX
++ cat /tmp/tmp.dSTRuEH9vd
++ rm /tmp/tmp.tzll9r9yUX /tmp/tmp.dSTRuEH9vd
++ return 0
+ status_query='kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zVUjf9ns0o
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9hw9fFhZOG
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.zVUjf9ns0o
+++ cat /tmp/tmp.9hw9fFhZOG
+++ rm /tmp/tmp.zVUjf9ns0o /tmp/tmp.9hw9fFhZOG
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.CBT32xurii
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SmTiYjcVQn
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.CBT32xurii
+++ cat /tmp/tmp.SmTiYjcVQn
+++ rm /tmp/tmp.CBT32xurii /tmp/tmp.SmTiYjcVQn
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.g5OfN2sg0H
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DPvAzMTgmX
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.g5OfN2sg0H
+++ cat /tmp/tmp.DPvAzMTgmX
+++ rm /tmp/tmp.g5OfN2sg0H /tmp/tmp.DPvAzMTgmX
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.avANe4zrVI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.nNtJjdsCsR
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.avANe4zrVI
+++ cat /tmp/tmp.nNtJjdsCsR
+++ rm /tmp/tmp.avANe4zrVI /tmp/tmp.nNtJjdsCsR
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.KEuIZo1kAd
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.ZF7d3D07k2
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.KEuIZo1kAd
++++ cat /tmp/tmp.ZF7d3D07k2
++++ rm /tmp/tmp.KEuIZo1kAd /tmp/tmp.ZF7d3D07k2
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.uIr88gGwRE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pUmgI2yXZZ
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.uIr88gGwRE
+++ cat /tmp/tmp.pUmgI2yXZZ
+++ rm /tmp/tmp.uIr88gGwRE /tmp/tmp.pUmgI2yXZZ
+++ return 0
++ echo postgres_password
+ run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.recreate-1535
+ local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.z8T6YPpuBd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.nOiMaEYbo8
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.z8T6YPpuBd
++ cat /tmp/tmp.nOiMaEYbo8
++ rm /tmp/tmp.z8T6YPpuBd /tmp/tmp.nOiMaEYbo8
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.6fy2aE7q8u
++ mktemp
+ local LAST_ERR=/tmp/tmp.BwtG5MkDbB
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6fy2aE7q8u
+ cat /tmp/tmp.BwtG5MkDbB
+ rm /tmp/tmp.6fy2aE7q8u /tmp/tmp.BwtG5MkDbB
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.c2ZGolm3ZV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xt15ml9YrP
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.c2ZGolm3ZV
+++ cat /tmp/tmp.xt15ml9YrP
+++ rm /tmp/tmp.c2ZGolm3ZV /tmp/tmp.xt15ml9YrP
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.FlqFcGOoz5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9ClUQ2zw1h
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.FlqFcGOoz5
+++ cat /tmp/tmp.9ClUQ2zw1h
+++ rm /tmp/tmp.FlqFcGOoz5 /tmp/tmp.9ClUQ2zw1h
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.iqpzB0wULD
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vlREnTT9zy
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.iqpzB0wULD
+++ cat /tmp/tmp.vlREnTT9zy
+++ rm /tmp/tmp.iqpzB0wULD /tmp/tmp.vlREnTT9zy
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.niEUKVT6pR
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.jqSM7IeVyQ
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.niEUKVT6pR
++++ cat /tmp/tmp.jqSM7IeVyQ
++++ rm /tmp/tmp.niEUKVT6pR /tmp/tmp.jqSM7IeVyQ
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.TgminRQPZn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ILOuZu97yC
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.TgminRQPZn
+++ cat /tmp/tmp.ILOuZu97yC
+++ rm /tmp/tmp.TgminRQPZn /tmp/tmp.ILOuZu97yC
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.recreate-1535
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zqErPcB1Ug
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YiK4UBRuTt
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.zqErPcB1Ug
++ cat /tmp/tmp.YiK4UBRuTt
++ rm /tmp/tmp.zqErPcB1Ug /tmp/tmp.YiK4UBRuTt
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UFK3ApQFzA
++ mktemp
+ local LAST_ERR=/tmp/tmp.irYJqXXgIe
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.UFK3ApQFzA
+ cat /tmp/tmp.irYJqXXgIe
+ rm /tmp/tmp.UFK3ApQFzA /tmp/tmp.irYJqXXgIe
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wUgbN3HLck
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.79z12AM26W
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.wUgbN3HLck
+++ cat /tmp/tmp.79z12AM26W
+++ rm /tmp/tmp.wUgbN3HLck /tmp/tmp.79z12AM26W
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MepeRVGPTP
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QRchIEBy59
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.MepeRVGPTP
+++ cat /tmp/tmp.QRchIEBy59
+++ rm /tmp/tmp.MepeRVGPTP /tmp/tmp.QRchIEBy59
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.DMpumEeb91
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7XbCsePKAH
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.DMpumEeb91
+++ cat /tmp/tmp.7XbCsePKAH
+++ rm /tmp/tmp.DMpumEeb91 /tmp/tmp.7XbCsePKAH
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.VNDbBxs7p6
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.pYu4tYSrPE
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.VNDbBxs7p6
++++ cat /tmp/tmp.pYu4tYSrPE
++++ rm /tmp/tmp.VNDbBxs7p6 /tmp/tmp.pYu4tYSrPE
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.whzC5336fe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QfGdSLhNam
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.whzC5336fe
+++ cat /tmp/tmp.QfGdSLhNam
+++ rm /tmp/tmp.whzC5336fe /tmp/tmp.QfGdSLhNam
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.recreate-1535
+ local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.CSzJ9RvrP1
+++ mktemp
++ local LAST_ERR=/tmp/tmp.CE7QPVgMs5
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.CSzJ9RvrP1
++ cat /tmp/tmp.CE7QPVgMs5
++ rm /tmp/tmp.CSzJ9RvrP1 /tmp/tmp.CE7QPVgMs5
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.YhUlzKA00v
++ mktemp
+ local LAST_ERR=/tmp/tmp.5cSnVJKAy7
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.YhUlzKA00v
+ cat /tmp/tmp.5cSnVJKAy7
+ rm /tmp/tmp.YhUlzKA00v /tmp/tmp.5cSnVJKAy7
+ return 0
+ sleep 10
+ desc 'Quick pgbouncer switch off and switch on'
+ set +o xtrace


-----------------------------------------------------------------------------------
Quick pgbouncer switch off and switch on
-----------------------------------------------------------------------------------

+ disable_pgBouncer some-name
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.8tgUoCGLvu
++ mktemp
+ local LAST_ERR=/tmp/tmp.pzGdu0SFBn
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":0}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.8tgUoCGLvu
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.pzGdu0SFBn
+ rm /tmp/tmp.8tgUoCGLvu /tmp/tmp.pzGdu0SFBn
+ return 0
+ kubectl_bin wait --timeout=300s --for=delete deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.icsbO3fdLn
++ mktemp
+ local LAST_ERR=/tmp/tmp.GUz7l3OSWG
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl wait --timeout=300s --for=delete deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.icsbO3fdLn
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.GUz7l3OSWG
+ rm /tmp/tmp.icsbO3fdLn /tmp/tmp.GUz7l3OSWG
+ return 0
+ enable_pgBouncer some-name 3
+ local cluster_name=some-name
+ local replicas_num=3
+ kubectl_bin patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Uprw1wYVke
++ mktemp
+ local LAST_ERR=/tmp/tmp.KfEZjwjblO
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json '-p=[{"op":"replace","path":"/spec/pgBouncer/size","value":3}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Uprw1wYVke
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.KfEZjwjblO
+ rm /tmp/tmp.Uprw1wYVke /tmp/tmp.KfEZjwjblO
+ return 0
+ sleep 60
+ kubectl_bin wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
++ mktemp
+ local LAST_OUT=/tmp/tmp.Le0wCJwuGj
++ mktemp
+ local LAST_ERR=/tmp/tmp.v3c7dO796N
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl wait --timeout=300s --for=condition=Available deployment/some-name-pgbouncer
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Le0wCJwuGj
deployment.apps/some-name-pgbouncer condition met
+ cat /tmp/tmp.v3c7dO796N
+ rm /tmp/tmp.Le0wCJwuGj /tmp/tmp.v3c7dO796N
+ return 0
+ wait_deployment some-name-pgbouncer
+ local name=some-name-pgbouncer
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-pgbouncer
++ get_psql_user_pass pgbouncer some-name
++ local user=pgbouncer
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VOWu6fhVRV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TUzlL7UDqw
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.VOWu6fhVRV
+++ cat /tmp/tmp.TUzlL7UDqw
+++ rm /tmp/tmp.VOWu6fhVRV /tmp/tmp.TUzlL7UDqw
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.bMROv2XigZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.XOvR6NRJo0
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.bMROv2XigZ
+++ cat /tmp/tmp.XOvR6NRJo0
+++ rm /tmp/tmp.bMROv2XigZ /tmp/tmp.XOvR6NRJo0
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.aga82lB09G
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.iGIfVwaT55
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.aga82lB09G
+++ cat /tmp/tmp.iGIfVwaT55
+++ rm /tmp/tmp.aga82lB09G /tmp/tmp.iGIfVwaT55
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.zGHfqUbsnz
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.t20EwAge8G
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.zGHfqUbsnz
++++ cat /tmp/tmp.t20EwAge8G
++++ rm /tmp/tmp.zGHfqUbsnz /tmp/tmp.t20EwAge8G
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wusNSaBPjm
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.KXA3iw3SsN
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.pgbouncer}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.wusNSaBPjm
+++ cat /tmp/tmp.KXA3iw3SsN
+++ rm /tmp/tmp.wusNSaBPjm /tmp/tmp.KXA3iw3SsN
+++ return 0
++ echo pgbouncer_password
+ compare_psql_cmd show-1 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535
+ local command_id=show-1
+ local 'command=SHOW DATABASES'
+ local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/show-1.sql
+ run_psql 'SHOW DATABASES' pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535 postgres ''
+ local 'command=SHOW DATABASES'
+ local uri=pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.tgIJIxtW72
+++ mktemp
++ local LAST_ERR=/tmp/tmp.QxOq47SNKU
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.tgIJIxtW72
++ cat /tmp/tmp.QxOq47SNKU
++ rm /tmp/tmp.tgIJIxtW72 /tmp/tmp.QxOq47SNKU
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.qpifbl7M2v
++ mktemp
+ local LAST_ERR=/tmp/tmp.0aOG604Akk
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SHOW DATABASES\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://pgbouncer:pgbouncer_password@some-name-pgbouncer.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.qpifbl7M2v
+ cat /tmp/tmp.0aOG604Akk
+ rm /tmp/tmp.qpifbl7M2v /tmp/tmp.0aOG604Akk
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/show-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/show-1.sql /tmp/tmp.u8o2J7yvV1/show-1.sql
++ get_psql_user_pass some-name some-name
++ local user=some-name
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.C1cpOZVgPl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dGJUw0xNiu
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.C1cpOZVgPl
+++ cat /tmp/tmp.dGJUw0xNiu
+++ rm /tmp/tmp.C1cpOZVgPl /tmp/tmp.dGJUw0xNiu
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.lFVLLOxQ1A
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LSsSlguMo0
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.lFVLLOxQ1A
+++ cat /tmp/tmp.LSsSlguMo0
+++ rm /tmp/tmp.lFVLLOxQ1A /tmp/tmp.LSsSlguMo0
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MtLzBwmGsB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YhRhw4tP1h
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.MtLzBwmGsB
+++ cat /tmp/tmp.YhRhw4tP1h
+++ rm /tmp/tmp.MtLzBwmGsB /tmp/tmp.YhRhw4tP1h
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.rKdDWnwHxs
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.YsqQTdEF5O
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.rKdDWnwHxs
++++ cat /tmp/tmp.YsqQTdEF5O
++++ rm /tmp/tmp.rKdDWnwHxs /tmp/tmp.YsqQTdEF5O
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.tRPDd3BB9f
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0ydUNIHQLe
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.tRPDd3BB9f
+++ cat /tmp/tmp.0ydUNIHQLe
+++ rm /tmp/tmp.tRPDd3BB9f /tmp/tmp.0ydUNIHQLe
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-1535
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-1535
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-pgbouncer.recreate-1535 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-pgbouncer.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.EXGXr53PdC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OC2n6Ky16k
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.EXGXr53PdC
++ cat /tmp/tmp.OC2n6Ky16k
++ rm /tmp/tmp.EXGXr53PdC /tmp/tmp.OC2n6Ky16k
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.LLMPlVadRc
++ mktemp
+ local LAST_ERR=/tmp/tmp.jq3xqyrOFQ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-pgbouncer.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.LLMPlVadRc
+ cat /tmp/tmp.jq3xqyrOFQ
+ rm /tmp/tmp.LLMPlVadRc /tmp/tmp.jq3xqyrOFQ
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.u8o2J7yvV1/select-1.sql
+ compare_kubectl deployment/some-name-pgbouncer
+ local resource=deployment/some-name-pgbouncer
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/deployment_some-name-pgbouncer.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin get -o yaml deployment/some-name-pgbouncer
+ yq d - '**.creationTimestamp'
+ yq d - '**.namespace'
++ mktemp
+ local LAST_OUT=/tmp/tmp.U5IAxELV0q
+ yq d - metadata.resourceVersion
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - metadata.selfLink
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ yq d - '**.uid'
+ yq d - metadata.deletionTimestamp
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**.creationTimestamp'
+ yq d - '**.image'
+ yq d - '**.procMount'
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - '**.clusterIP'
+ yq d - '**.storageClassName'
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - '**.finalizers'
+ yq d - '**.volumeName'
+ yq d - '**.enableServiceLinks'
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
++ mktemp
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - spec.volumeMode
+ yq d - '**.controller-uid'
+ local LAST_ERR=/tmp/tmp.aNJTBE9aBL
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ yq d - spec.nodeName
+ yq d - '**.nodePort'
+ yq d - '**.(name==suffix)'
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - '**.imagePullSecrets'
+ yq d - '**.ipFamilies'
+ yq d - status
+ yq d - '**.clusterIPs'
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.dataSource'
+ yq d - '**."percona.com/*"'
+ yq d - '**.ipFamilyPolicy'
+ yq d - '**.internalTrafficPolicy'
+ yq d - '**.(name==PODNAME)'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - '**.preemptionPolicy'
+ yq d - '**.healthCheckNodePort'
+ yq d - 'metadata.ownerReferences.*.apiVersion'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml deployment/some-name-pgbouncer
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - metadata.managedFields
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.U5IAxELV0q
+ cat /tmp/tmp.aNJTBE9aBL
+ rm /tmp/tmp.U5IAxELV0q /tmp/tmp.aNJTBE9aBL
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/deployment_some-name-pgbouncer.yml kind
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/deployment_some-name-pgbouncer.yml /tmp/tmp.u8o2J7yvV1/deployment_some-name-pgbouncer.yml
+ compare_kubectl service/some-name-pgbouncer
+ local resource=service/some-name-pgbouncer
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq d - '**.creationTimestamp'
+ yq d - '**.namespace'
+ kubectl_bin get -o yaml service/some-name-pgbouncer
+ yq d - metadata.deletionTimestamp
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - '**.uid'
++ mktemp
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**.creationTimestamp'
+ local LAST_OUT=/tmp/tmp.BmJ4Hkshro
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ yq d - '**.image'
+ yq d - '**.clusterIP'
+ yq d - '**.clusterIPs'
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - '**.storageClassName'
+ yq d - metadata.selfLink
+ yq d - '**.procMount'
+ yq d - metadata.resourceVersion
+ yq d - '**.finalizers'
+ yq d - spec.nodeName
+ yq d - '**.imagePullSecrets'
++ mktemp
+ yq d - '**.enableServiceLinks'
+ yq d - '**.(name==suffix)'
+ yq d - '**.dataSource'
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ yq d - status
+ yq d - '**.ipFamilies'
+ yq d - '**.volumeName'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
+ local LAST_ERR=/tmp/tmp.eHbI3Bns7x
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - spec.volumeMode
+ yq d - '**."percona.com/*"'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ yq d - '**.healthCheckNodePort'
+ yq d - metadata.managedFields
+ yq d - '**.nodePort'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml service/some-name-pgbouncer
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - '**.(name==PODNAME)'
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - 'metadata.ownerReferences.*.apiVersion'
+ yq d - '**.controller-uid'
+ yq d - '**.preemptionPolicy'
+ yq d - '**.ipFamilyPolicy'
+ yq d - '**.internalTrafficPolicy'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.BmJ4Hkshro
+ cat /tmp/tmp.eHbI3Bns7x
+ rm /tmp/tmp.BmJ4Hkshro /tmp/tmp.eHbI3Bns7x
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml kind
++ yq r /tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml metadata.annotations
+ [[ x{} == \x ]]
++ yq r /tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml metadata.annotations
+ [[ x{} == \x\{\} ]]
+ yq d -i /tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml metadata.annotations
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/service_some-name-pgbouncer.yml /tmp/tmp.u8o2J7yvV1/service_some-name-pgbouncer.yml
++ seq 1 3
+ for repl in '$(seq 1 3)'
+ desc 'check replica 1'
+ set +o xtrace


-----------------------------------------------------------------------------------
check replica 1
-----------------------------------------------------------------------------------

+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.70MiVZMcWZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.8B4zl6xqdO
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.70MiVZMcWZ
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.8B4zl6xqdO
+ rm /tmp/tmp.70MiVZMcWZ /tmp/tmp.8B4zl6xqdO
+ return 0
+ sleep 30
+ wait_deployment some-name-repl1
+ local name=some-name-repl1
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl1
+ sleep 30
+ check_replica some-name-repl1 some-name select-1
+ local name=some-name-repl1
+ local cluster=some-name
+ local sql_cmp=select-1
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Sma0u1FP8B
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Z8i1bxYE7x
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.Sma0u1FP8B
+++ cat /tmp/tmp.Z8i1bxYE7x
+++ rm /tmp/tmp.Sma0u1FP8B /tmp/tmp.Z8i1bxYE7x
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.stXq3IvSFE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.EpldL2sTlp
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.stXq3IvSFE
+++ cat /tmp/tmp.EpldL2sTlp
+++ rm /tmp/tmp.stXq3IvSFE /tmp/tmp.EpldL2sTlp
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LY7oHuzoYX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4iC8m1kaUz
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.LY7oHuzoYX
+++ cat /tmp/tmp.4iC8m1kaUz
+++ rm /tmp/tmp.LY7oHuzoYX /tmp/tmp.4iC8m1kaUz
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.0QRgPhHB73
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.E4G770DfWq
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.0QRgPhHB73
++++ cat /tmp/tmp.E4G770DfWq
++++ rm /tmp/tmp.0QRgPhHB73 /tmp/tmp.E4G770DfWq
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.V4vhPLYSIw
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.gRTtx7pEuW
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.V4vhPLYSIw
+++ cat /tmp/tmp.gRTtx7pEuW
+++ rm /tmp/tmp.V4vhPLYSIw /tmp/tmp.gRTtx7pEuW
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.1PFtrGJzte
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YY4EomlBWD
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.1PFtrGJzte
++ cat /tmp/tmp.YY4EomlBWD
++ rm /tmp/tmp.1PFtrGJzte /tmp/tmp.YY4EomlBWD
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NEXJqt8ELs
++ mktemp
+ local LAST_ERR=/tmp/tmp.EPmDYNsYZG
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.NEXJqt8ELs
+ cat /tmp/tmp.EPmDYNsYZG
+ rm /tmp/tmp.NEXJqt8ELs /tmp/tmp.EPmDYNsYZG
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ShXvaZGCqP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JI7MgWAijn
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.ShXvaZGCqP
++ cat /tmp/tmp.JI7MgWAijn
++ rm /tmp/tmp.ShXvaZGCqP /tmp/tmp.JI7MgWAijn
++ return 0
+ replica_pod_name=some-name-repl1-7987df9b7-6wxhr
++ grep some-name-repl1-7987df9b7-6wxhr /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl1-7987df9b7-6wxhr | 10.55.248.24 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.sKw8GnADcZ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bAO8LK1Sio
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.sKw8GnADcZ
++ cat /tmp/tmp.bAO8LK1Sio
++ rm /tmp/tmp.sKw8GnADcZ /tmp/tmp.bAO8LK1Sio
++ return 0
+ replica_pod_IP=10.55.248.24
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.248.24 '' ' '
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.248.24
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.248.24 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.248.24
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MpZ75RcEXN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8DPnVs4Suk
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.MpZ75RcEXN
++ cat /tmp/tmp.8DPnVs4Suk
++ rm /tmp/tmp.MpZ75RcEXN /tmp/tmp.8DPnVs4Suk
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.248.24 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.pHwc7wOA5T
++ mktemp
+ local LAST_ERR=/tmp/tmp.8xSbLOCcrC
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.248.24 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.pHwc7wOA5T
+ cat /tmp/tmp.8xSbLOCcrC
+ rm /tmp/tmp.pHwc7wOA5T /tmp/tmp.8xSbLOCcrC
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.u8o2J7yvV1/select-1.sql
+ for repl in '$(seq 1 3)'
+ desc 'check replica 2'
+ set +o xtrace


-----------------------------------------------------------------------------------
check replica 2
-----------------------------------------------------------------------------------

+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.pgrKOSgeFb
++ mktemp
+ local LAST_ERR=/tmp/tmp.PT7Z5Aa7kW
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.pgrKOSgeFb
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.PT7Z5Aa7kW
+ rm /tmp/tmp.pgrKOSgeFb /tmp/tmp.PT7Z5Aa7kW
+ return 0
+ sleep 30
+ wait_deployment some-name-repl2
+ local name=some-name-repl2
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl2
+ sleep 30
+ check_replica some-name-repl2 some-name select-1
+ local name=some-name-repl2
+ local cluster=some-name
+ local sql_cmp=select-1
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.74cIZ75TJQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rewsQeoy3p
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.74cIZ75TJQ
+++ cat /tmp/tmp.rewsQeoy3p
+++ rm /tmp/tmp.74cIZ75TJQ /tmp/tmp.rewsQeoy3p
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ENO7egT4yA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Xpx5XFGTOw
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ENO7egT4yA
+++ cat /tmp/tmp.Xpx5XFGTOw
+++ rm /tmp/tmp.ENO7egT4yA /tmp/tmp.Xpx5XFGTOw
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kKPZ4AY0bl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.mCfjEhqRo2
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.kKPZ4AY0bl
+++ cat /tmp/tmp.mCfjEhqRo2
+++ rm /tmp/tmp.kKPZ4AY0bl /tmp/tmp.mCfjEhqRo2
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.tY6IGAgcXf
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.gQ0OwtSRo8
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.tY6IGAgcXf
++++ cat /tmp/tmp.gQ0OwtSRo8
++++ rm /tmp/tmp.tY6IGAgcXf /tmp/tmp.gQ0OwtSRo8
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SaWbia6A61
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wN5yr2fl3s
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.SaWbia6A61
+++ cat /tmp/tmp.wN5yr2fl3s
+++ rm /tmp/tmp.SaWbia6A61 /tmp/tmp.wN5yr2fl3s
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qrQ5dvSSNp
+++ mktemp
++ local LAST_ERR=/tmp/tmp.z9Wwi3bPu8
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.qrQ5dvSSNp
++ cat /tmp/tmp.z9Wwi3bPu8
++ rm /tmp/tmp.qrQ5dvSSNp /tmp/tmp.z9Wwi3bPu8
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4dwRGQc1wz
++ mktemp
+ local LAST_ERR=/tmp/tmp.1A6maI7Ylt
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.4dwRGQc1wz
+ cat /tmp/tmp.1A6maI7Ylt
+ rm /tmp/tmp.4dwRGQc1wz /tmp/tmp.1A6maI7Ylt
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.FA0B0a3r7s
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UICVsIivY6
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.FA0B0a3r7s
++ cat /tmp/tmp.UICVsIivY6
++ rm /tmp/tmp.FA0B0a3r7s /tmp/tmp.UICVsIivY6
++ return 0
+ replica_pod_name=some-name-repl2-57cf7d6dcf-48dr6
++ grep some-name-repl2-57cf7d6dcf-48dr6 /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl2-57cf7d6dcf-48dr6 | 10.55.250.62 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ENalgQxanX
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JArjTO8kJF
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.ENalgQxanX
++ cat /tmp/tmp.JArjTO8kJF
++ rm /tmp/tmp.ENalgQxanX /tmp/tmp.JArjTO8kJF
++ return 0
+ replica_pod_IP=10.55.250.62
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.250.62 '' ' '
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.250.62
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.250.62 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.250.62
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.INrOavP4df
+++ mktemp
++ local LAST_ERR=/tmp/tmp.FZ8P2A3DTU
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.INrOavP4df
++ cat /tmp/tmp.FZ8P2A3DTU
++ rm /tmp/tmp.INrOavP4df /tmp/tmp.FZ8P2A3DTU
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.250.62 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.iwOvbn2MWp
++ mktemp
+ local LAST_ERR=/tmp/tmp.MFyIqirMjt
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.250.62 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.iwOvbn2MWp
+ cat /tmp/tmp.MFyIqirMjt
+ rm /tmp/tmp.iwOvbn2MWp /tmp/tmp.MFyIqirMjt
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.u8o2J7yvV1/select-1.sql
+ for repl in '$(seq 1 3)'
+ desc 'check replica 3'
+ set +o xtrace


-----------------------------------------------------------------------------------
check replica 3
-----------------------------------------------------------------------------------

+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.cXwnIzfV2r
++ mktemp
+ local LAST_ERR=/tmp/tmp.1GaJRPoYDN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.cXwnIzfV2r
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.1GaJRPoYDN
+ rm /tmp/tmp.cXwnIzfV2r /tmp/tmp.1GaJRPoYDN
+ return 0
+ sleep 30
+ wait_deployment some-name-repl3
+ local name=some-name-repl3
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl3
+ sleep 30
+ check_replica some-name-repl3 some-name select-1
+ local name=some-name-repl3
+ local cluster=some-name
+ local sql_cmp=select-1
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OuVIWwS4GB
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.0NGtwasKX3
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.OuVIWwS4GB
+++ cat /tmp/tmp.0NGtwasKX3
+++ rm /tmp/tmp.OuVIWwS4GB /tmp/tmp.0NGtwasKX3
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.hnYmuigTrQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7CG0bSMK7z
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.hnYmuigTrQ
+++ cat /tmp/tmp.7CG0bSMK7z
+++ rm /tmp/tmp.hnYmuigTrQ /tmp/tmp.7CG0bSMK7z
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.OeL3sNAeHy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.e1Ol5sqIkN
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.OeL3sNAeHy
+++ cat /tmp/tmp.e1Ol5sqIkN
+++ rm /tmp/tmp.OeL3sNAeHy /tmp/tmp.e1Ol5sqIkN
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.EOVdRABAq9
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.mMSDTxA4xE
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.EOVdRABAq9
++++ cat /tmp/tmp.mMSDTxA4xE
++++ rm /tmp/tmp.EOVdRABAq9 /tmp/tmp.mMSDTxA4xE
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.XcRl0whpre
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FaJ1K30YMV
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.XcRl0whpre
+++ cat /tmp/tmp.FaJ1K30YMV
+++ rm /tmp/tmp.XcRl0whpre /tmp/tmp.FaJ1K30YMV
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k37vtbwh7Y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.J4LkkTKi4G
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.k37vtbwh7Y
++ cat /tmp/tmp.J4LkkTKi4G
++ rm /tmp/tmp.k37vtbwh7Y /tmp/tmp.J4LkkTKi4G
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.vvcoDE4UoP
++ mktemp
+ local LAST_ERR=/tmp/tmp.RMMO7TCxy9
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.vvcoDE4UoP
+ cat /tmp/tmp.RMMO7TCxy9
+ rm /tmp/tmp.vvcoDE4UoP /tmp/tmp.RMMO7TCxy9
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TnMnq2ZNl5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.xyArbvFoEN
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.TnMnq2ZNl5
++ cat /tmp/tmp.xyArbvFoEN
++ rm /tmp/tmp.TnMnq2ZNl5 /tmp/tmp.xyArbvFoEN
++ return 0
+ replica_pod_name=some-name-repl3-97b8b7b55-jn56s
++ grep some-name-repl3-97b8b7b55-jn56s /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl3-97b8b7b55-jn56s  | 10.55.249.10 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.OoByHPheQW
+++ mktemp
++ local LAST_ERR=/tmp/tmp.aTHMNGckSU
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.OoByHPheQW
++ cat /tmp/tmp.aTHMNGckSU
++ rm /tmp/tmp.OoByHPheQW /tmp/tmp.aTHMNGckSU
++ return 0
+ replica_pod_IP=10.55.249.10
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.249.10 '' ' '
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.249.10
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.249.10 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.249.10
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.jgmspygxwc
+++ mktemp
++ local LAST_ERR=/tmp/tmp.a4FOYM8QAX
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.jgmspygxwc
++ cat /tmp/tmp.a4FOYM8QAX
++ rm /tmp/tmp.jgmspygxwc /tmp/tmp.a4FOYM8QAX
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.249.10 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.UGAam5kijA
++ mktemp
+ local LAST_ERR=/tmp/tmp.Py3wpDuVCQ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.249.10 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.UGAam5kijA
+ cat /tmp/tmp.Py3wpDuVCQ
+ rm /tmp/tmp.UGAam5kijA /tmp/tmp.Py3wpDuVCQ
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.u8o2J7yvV1/select-1.sql
+ desc 'Restart the cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
Restart the cluster
-----------------------------------------------------------------------------------

+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.KsLM3WlRuQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.pgIpDJkS0Y
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepData","value":true}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.KsLM3WlRuQ
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.pgIpDJkS0Y
+ rm /tmp/tmp.KsLM3WlRuQ /tmp/tmp.pgIpDJkS0Y
+ return 0
+ sleep 30
++ seq 1 3
+ for i in '$(seq 1 3)'
+ sleep 15
+ wait_deployment some-name-repl1
+ local name=some-name-repl1
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl1...
+ for i in '$(seq 1 3)'
+ sleep 15
+ wait_deployment some-name-repl2
+ local name=some-name-repl2
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl2.........
+ for i in '$(seq 1 3)'
+ sleep 15
+ wait_deployment some-name-repl3
+ local name=some-name-repl3
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl3...........
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name........
+ sleep 30
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.VnAh2jcHFu
++ mktemp
+ local LAST_ERR=/tmp/tmp.KQdzc1ejfw
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.VnAh2jcHFu
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.KQdzc1ejfw
+ rm /tmp/tmp.VnAh2jcHFu /tmp/tmp.KQdzc1ejfw
+ return 0
+ wait_for_delete pgreplica/some-name-repl3
+ local res=pgreplica/some-name-repl3
+ set +o xtrace
pgreplica/some-name-repl3 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl3" not found
+ sleep 30
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.eJ1qxXqVru
++ mktemp
+ local LAST_ERR=/tmp/tmp.WDSUZ8bKyI
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.eJ1qxXqVru
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.WDSUZ8bKyI
+ rm /tmp/tmp.eJ1qxXqVru /tmp/tmp.WDSUZ8bKyI
+ return 0
+ wait_for_delete pgreplica/some-name-repl2
+ local res=pgreplica/some-name-repl2
+ set +o xtrace
pgreplica/some-name-repl2 - ....Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl2" not found
+ sleep 30
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.9Y8P4GvWJT
++ mktemp
+ local LAST_ERR=/tmp/tmp.xL6BfAX2fm
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 0}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.9Y8P4GvWJT
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.xL6BfAX2fm
+ rm /tmp/tmp.9Y8P4GvWJT /tmp/tmp.xL6BfAX2fm
+ return 0
+ wait_for_delete pgreplica/some-name-repl1
+ local res=pgreplica/some-name-repl1
+ set +o xtrace
pgreplica/some-name-repl1 - .Error from server (NotFound): pgreplicas.pg.percona.com "some-name-repl1" not found
+ sleep 30
+ kubectl_bin delete perconapgclusters --all
++ mktemp
+ local LAST_OUT=/tmp/tmp.6i7MvNCYys
++ mktemp
+ local LAST_ERR=/tmp/tmp.Afw1nfTOBX
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete perconapgclusters --all
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6i7MvNCYys
perconapgcluster.pg.percona.com "some-name" deleted
+ cat /tmp/tmp.Afw1nfTOBX
+ rm /tmp/tmp.6i7MvNCYys /tmp/tmp.Afw1nfTOBX
+ return 0
+ wait_for_delete pgcluster/some-name
+ local res=pgcluster/some-name
+ set +o xtrace
pgcluster/some-name - Error from server (NotFound): pgclusters.pg.percona.com "some-name" not found
+ wait_for_delete deployment/some-name
+ local res=deployment/some-name
+ set +o xtrace
deployment/some-name - Error from server (NotFound): deployments.apps "some-name" not found
+ wait_for_delete deployment/some-name-backrest-shared-repo
+ local res=deployment/some-name-backrest-shared-repo
+ set +o xtrace
deployment/some-name-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found
+ compare_kubectl pvc/some-name
+ local resource=pvc/some-name
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/pvc_some-name.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin get -o yaml pvc/some-name
+ yq d - '**.creationTimestamp'
+ yq d - metadata.managedFields
++ mktemp
+ yq d - metadata.resourceVersion
+ yq d - '**.namespace'
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ local LAST_OUT=/tmp/tmp.tblK6hMiz9
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**.uid'
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - '**.creationTimestamp'
+ yq d - '**.clusterIP'
+ yq d - '**.volumeName'
+ yq d - '**.clusterIPs'
++ mktemp
+ yq d - '**.storageClassName'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ local LAST_ERR=/tmp/tmp.CqyhhiDqXH
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ yq d - status
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - '**.healthCheckNodePort'
+ yq d - '**.nodePort'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - '**."percona.com/*"'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ yq d - '**.(name==suffix)'
+ yq d - '**.dataSource'
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.imagePullSecrets'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
+ yq d - '**.procMount'
++ seq 0 2
+ yq d - '**.finalizers'
+ yq d - '**.controller-uid'
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - '**.ipFamilies'
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml pvc/some-name
+ yq d - spec.nodeName
+ yq d - '**.ipFamilyPolicy'
+ yq d - '**.enableServiceLinks'
+ yq d - '**.(name==PODNAME)'
+ yq d - '**.internalTrafficPolicy'
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - spec.volumeMode
+ yq d - 'metadata.ownerReferences.*.apiVersion'
+ yq d - '**.preemptionPolicy'
+ yq d - '**.image'
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - metadata.deletionTimestamp
+ yq d - metadata.selfLink
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.tblK6hMiz9
+ cat /tmp/tmp.CqyhhiDqXH
+ rm /tmp/tmp.tblK6hMiz9 /tmp/tmp.CqyhhiDqXH
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/pvc_some-name.yml kind
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name.yml /tmp/tmp.u8o2J7yvV1/pvc_some-name.yml
+ compare_kubectl pvc/some-name-repl1
+ local resource=pvc/some-name-repl1
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl1.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/pvc_some-name-repl1.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq d - metadata.managedFields
+ yq d - '**.namespace'
+ yq d - '**.creationTimestamp'
+ yq d - '**.uid'
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - '**.clusterIP'
+ yq d - metadata.deletionTimestamp
+ yq d - '**.clusterIPs'
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - metadata.resourceVersion
+ yq d - '**.dataSource'
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - '**.procMount'
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ yq d - spec.volumeMode
+ yq d - '**.nodePort'
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ yq d - '**."percona.com/*"'
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - '**.enableServiceLinks'
+ yq d - spec.nodeName
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ yq d - '**.imagePullSecrets'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ yq d - status
+ yq d - '**.(name==suffix)'
+ yq d - '**.(name==PODNAME)'
+ yq d - '**.healthCheckNodePort'
+ yq d - '**.image'
+ yq d - '**.controller-uid'
+ yq d - '**.preemptionPolicy'
+ yq d - 'metadata.ownerReferences.*.apiVersion'
+ yq d - '**.storageClassName'
+ yq d - '**.finalizers'
+ yq d - '**.ipFamilies'
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - '**.ipFamilyPolicy'
+ yq d - '**.volumeName'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - '**.internalTrafficPolicy'
+ yq d - '**.creationTimestamp'
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - metadata.selfLink
+ kubectl_bin get -o yaml pvc/some-name-repl1
++ mktemp
+ local LAST_OUT=/tmp/tmp.ahEuXCdyDV
++ mktemp
+ local LAST_ERR=/tmp/tmp.JLrWpW1uFo
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml pvc/some-name-repl1
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ahEuXCdyDV
+ cat /tmp/tmp.JLrWpW1uFo
+ rm /tmp/tmp.ahEuXCdyDV /tmp/tmp.JLrWpW1uFo
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl1.yml kind
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl1.yml /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl1.yml
+ compare_kubectl pvc/some-name-repl2
+ local resource=pvc/some-name-repl2
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl2.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/pvc_some-name-repl2.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq d - metadata.managedFields
+ yq d - '**.creationTimestamp'
+ yq d - '**.uid'
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - '**.namespace'
+ yq d - metadata.resourceVersion
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - metadata.selfLink
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - metadata.deletionTimestamp
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**.creationTimestamp'
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - '**.clusterIPs'
+ yq d - '**.dataSource'
+ yq d - '**.image'
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - '**.procMount'
+ yq d - '**.clusterIP'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - '**.(name==PODNAME)'
+ yq d - '**.storageClassName'
+ yq d - '**.volumeName'
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - spec.volumeMode
+ yq d - '**.finalizers'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
+ yq d - '**.healthCheckNodePort'
+ yq d - spec.nodeName
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ yq d - '**.imagePullSecrets'
+ yq d - 'metadata.ownerReferences.*.apiVersion'
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.enableServiceLinks'
+ yq d - '**.nodePort'
+ yq d - '**.controller-uid'
+ yq d - '**.preemptionPolicy'
+ yq d - '**.ipFamilies'
+ yq d - '**."percona.com/*"'
+ yq d - '**.ipFamilyPolicy'
+ yq d - '**.internalTrafficPolicy'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ yq d - '**.(name==suffix)'
+ yq d - status
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ kubectl_bin get -o yaml pvc/some-name-repl2
++ mktemp
+ local LAST_OUT=/tmp/tmp.04vrHHAPMm
++ mktemp
+ local LAST_ERR=/tmp/tmp.vSwdtrGVZc
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml pvc/some-name-repl2
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.04vrHHAPMm
+ cat /tmp/tmp.vSwdtrGVZc
+ rm /tmp/tmp.04vrHHAPMm /tmp/tmp.vSwdtrGVZc
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl2.yml kind
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl2.yml /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl2.yml
+ compare_kubectl pvc/some-name-repl3
+ local resource=pvc/some-name-repl3
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl3.yml
+ local new_result=/tmp/tmp.u8o2J7yvV1/pvc_some-name-repl3.yml
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq d - metadata.managedFields
+ yq d - '**.creationTimestamp'
+ yq d - '**.namespace'
+ yq d - '**.uid'
+ yq d - 'metadata.annotations."kubernetes.io/psp"'
+ yq d - metadata.selfLink
+ yq d - metadata.resourceVersion
+ yq d - 'metadata.annotations."cloud.google.com/neg"'
+ yq d - 'metadata.annotations."batch.kubernetes.io/job-tracking"'
+ yq d - '**."batch.kubernetes.io/controller-uid"'
+ yq d - 'spec.template.metadata.labels."batch.kubernetes.io/job-name"'
+ yq d - '**.creationTimestamp'
+ yq d - '**.image'
+ yq d - '**.clusterIP'
+ yq d - '**."volume.beta.kubernetes.io/storage-provisioner"'
+ yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"'
+ yq d - '**.finalizers'
+ yq d - '**."volume.kubernetes.io/storage-provisioner"'
+ yq d - '**.clusterIPs'
+ yq d - '**."kubernetes.io/pvc-protection"'
+ yq d - spec.volumeMode
+ yq d - metadata.deletionTimestamp
+ yq d - '**.procMount'
+ yq d - '**.volumeName'
+ yq d - '**.storageClassName'
+ yq d - '**."volume.kubernetes.io/selected-node"'
+ yq d - '**."percona.com/*"'
+ yq d - spec.nodeName
+ yq d - '**.dataSource'
+ yq d - '**.healthCheckNodePort'
+ yq d - 'spec.volumeClaimTemplates.*.apiVersion'
+ yq d - '**.(volumeMode==Filesystem).volumeMode'
+ yq d - '**.nodePort'
+ yq d - '**.imagePullSecrets'
+ yq d - '**.allocateLoadBalancerNodePorts'
+ yq d - '**.(name==PODNAME)'
+ yq d - '**.enableServiceLinks'
+ yq d - status
+ yq d - 'spec.volumeClaimTemplates.*.kind'
+ yq d - '**.(name==suffix)'
+ yq d - '**.ipFamilies'
+ yq d - '**.(name==NAMESPACE)'
+ yq d - '**.(name==COMMAND_OPTS)'
+ yq d - '**.ipFamilyPolicy'
+ yq d - 'metadata.ownerReferences.*.apiVersion'
+ yq d - '**.controller-uid'
+ yq d - '**.preemptionPolicy'
+ yq d - '**.internalTrafficPolicy'
+ kubectl_bin get -o yaml pvc/some-name-repl3
++ mktemp
+ local LAST_OUT=/tmp/tmp.268dUYYZS7
++ mktemp
+ local LAST_ERR=/tmp/tmp.yUuN1cNq9K
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get -o yaml pvc/some-name-repl3
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.268dUYYZS7
+ cat /tmp/tmp.yUuN1cNq9K
+ rm /tmp/tmp.268dUYYZS7 /tmp/tmp.yUuN1cNq9K
+ return 0
+ case $(yq r ${new_result} 'kind') in
++ yq r /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl3.yml kind
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/pvc_some-name-repl3.yml /tmp/tmp.u8o2J7yvV1/pvc_some-name-repl3.yml
+ sleep 30
+ create_user_secrets some-name
+ local cluster=some-name
+ local namespace=recreate-1535
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - stringData.some-name some-name_pass
+ yq w - metadata.name some-name-users-secret
+ kubectl_bin -n recreate-1535 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.bjdPL2sQ9V
++ mktemp
+ local LAST_ERR=/tmp/tmp.ZF9rxTkTgH
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.bjdPL2sQ9V
secret/some-name-users-secret configured
+ cat /tmp/tmp.ZF9rxTkTgH
+ rm /tmp/tmp.bjdPL2sQ9V /tmp/tmp.ZF9rxTkTgH
+ return 0
+ yq w - metadata.name some-name-pmm-secret
+ kubectl_bin -n recreate-1535 apply -f -
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.a5DawlXjZh
++ mktemp
+ local LAST_ERR=/tmp/tmp.9ZH1GvwU1v
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.a5DawlXjZh
secret/some-name-pmm-secret configured
+ cat /tmp/tmp.9ZH1GvwU1v
+ rm /tmp/tmp.a5DawlXjZh /tmp/tmp.9ZH1GvwU1v
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=
+ local schedule=
+ local schedule_type=
+ local tls_only=
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.labels.deployment-name some-name
+ yq w - metadata.labels.name some-name
+ yq w - spec.keepData false
+ yq w - spec.database some-name
+ yq w - spec.name some-name
+ yq w - spec.keepBackups false
+ yq w - spec.user some-name
+ yq w - spec.clustername some-name
+ yq w - metadata.name some-name
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ yq w - spec.secretsName some-name-users-secret
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ '' == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.u8o2J7yvV1/cr.yaml -n recreate-1535
++ mktemp
+ local LAST_OUT=/tmp/tmp.Oo313VxmI2
++ mktemp
+ local LAST_ERR=/tmp/tmp.wiHkJftdTu
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.u8o2J7yvV1/cr.yaml -n recreate-1535
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Oo313VxmI2
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.wiHkJftdTu
+ rm /tmp/tmp.Oo313VxmI2 /tmp/tmp.wiHkJftdTu
+ return 0
+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=recreate-1535
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GyabcNjM3Q
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UU5zE3inhn
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.GyabcNjM3Q
++ cat /tmp/tmp.UU5zE3inhn
++ rm /tmp/tmp.GyabcNjM3Q /tmp/tmp.UU5zE3inhn
++ return 0
+ status_query='kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2ButzG6mtr
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.eq44RqNI4K
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.2ButzG6mtr
+++ cat /tmp/tmp.eq44RqNI4K
+++ rm /tmp/tmp.2ButzG6mtr /tmp/tmp.eq44RqNI4K
+++ return 0
+ [[ '' == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 1 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.UMU0s5pLrs
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.FVrQJu9Qm6
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.UMU0s5pLrs
+++ cat /tmp/tmp.FVrQJu9Qm6
+++ rm /tmp/tmp.UMU0s5pLrs /tmp/tmp.FVrQJu9Qm6
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 2 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.NqQxi23Ozc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.4P6jMwtVMO
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.NqQxi23Ozc
+++ cat /tmp/tmp.4P6jMwtVMO
+++ rm /tmp/tmp.NqQxi23Ozc /tmp/tmp.4P6jMwtVMO
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 3 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JZqwCYer4R
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.RtVnkWeTUA
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.JZqwCYer4R
+++ cat /tmp/tmp.RtVnkWeTUA
+++ rm /tmp/tmp.JZqwCYer4R /tmp/tmp.RtVnkWeTUA
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 4 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.v6PJrDlvf0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.pL9M2yCzpa
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.v6PJrDlvf0
+++ cat /tmp/tmp.pL9M2yCzpa
+++ rm /tmp/tmp.v6PJrDlvf0 /tmp/tmp.pL9M2yCzpa
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 5 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.M7NGV6uiEC
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ThFFmw2sA1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.M7NGV6uiEC
+++ cat /tmp/tmp.ThFFmw2sA1
+++ rm /tmp/tmp.M7NGV6uiEC /tmp/tmp.ThFFmw2sA1
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 6 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jPQg161AHK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LpMfkkwIo4
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.jPQg161AHK
+++ cat /tmp/tmp.LpMfkkwIo4
+++ rm /tmp/tmp.jPQg161AHK /tmp/tmp.LpMfkkwIo4
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 7 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.h35yX0Atlc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kV0pTVihO7
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.h35yX0Atlc
+++ cat /tmp/tmp.kV0pTVihO7
+++ rm /tmp/tmp.h35yX0Atlc /tmp/tmp.kV0pTVihO7
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 8 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.APKb2PwYyI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.WceIJGTlT3
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.APKb2PwYyI
+++ cat /tmp/tmp.WceIJGTlT3
+++ rm /tmp/tmp.APKb2PwYyI /tmp/tmp.WceIJGTlT3
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 9 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vVf7dopeyI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.803wF5kZoJ
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.vVf7dopeyI
+++ cat /tmp/tmp.803wF5kZoJ
+++ rm /tmp/tmp.vVf7dopeyI /tmp/tmp.803wF5kZoJ
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 20
++ get_psql_user_pass some-name some-name
++ local user=some-name
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HU5r7Z5CQd
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.T5Ofm4M35o
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.HU5r7Z5CQd
+++ cat /tmp/tmp.T5Ofm4M35o
+++ rm /tmp/tmp.HU5r7Z5CQd /tmp/tmp.T5Ofm4M35o
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YY9vzXLYZ8
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.LaFZ6f9D1A
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.YY9vzXLYZ8
+++ cat /tmp/tmp.LaFZ6f9D1A
+++ rm /tmp/tmp.YY9vzXLYZ8 /tmp/tmp.LaFZ6f9D1A
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YyZGpxiMmQ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.2NE0qSd6PH
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.YyZGpxiMmQ
+++ cat /tmp/tmp.2NE0qSd6PH
+++ rm /tmp/tmp.YyZGpxiMmQ /tmp/tmp.2NE0qSd6PH
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.ifw3LbEDSn
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.FuoFT2rvMo
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.ifw3LbEDSn
++++ cat /tmp/tmp.FuoFT2rvMo
++++ rm /tmp/tmp.ifw3LbEDSn /tmp/tmp.FuoFT2rvMo
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.1V8p3VAvTf
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vEZhjPnmYZ
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.1V8p3VAvTf
+++ cat /tmp/tmp.vEZhjPnmYZ
+++ rm /tmp/tmp.1V8p3VAvTf /tmp/tmp.vEZhjPnmYZ
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-1535
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name.recreate-1535
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name.recreate-1535 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LkAgXo8fse
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lLr0fuUbZm
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.LkAgXo8fse
++ cat /tmp/tmp.lLr0fuUbZm
++ rm /tmp/tmp.LkAgXo8fse /tmp/tmp.lLr0fuUbZm
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.DZaafWxHhV
++ mktemp
+ local LAST_ERR=/tmp/tmp.sI49cxpWiG
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.DZaafWxHhV
+ cat /tmp/tmp.sI49cxpWiG
+ rm /tmp/tmp.DZaafWxHhV /tmp/tmp.sI49cxpWiG
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.u8o2J7yvV1/select-1.sql
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.4mtAolCuTU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OqDhicDI42
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.4mtAolCuTU
+++ cat /tmp/tmp.OqDhicDI42
+++ rm /tmp/tmp.4mtAolCuTU /tmp/tmp.OqDhicDI42
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.7i23KUdUDG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.epGmzVdQ1U
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.7i23KUdUDG
+++ cat /tmp/tmp.epGmzVdQ1U
+++ rm /tmp/tmp.7i23KUdUDG /tmp/tmp.epGmzVdQ1U
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.62anCWJEXt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.BUuapOCsgv
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.62anCWJEXt
+++ cat /tmp/tmp.BUuapOCsgv
+++ rm /tmp/tmp.62anCWJEXt /tmp/tmp.BUuapOCsgv
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.WwnFomIb7M
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.eCJEPlm3sA
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.WwnFomIb7M
++++ cat /tmp/tmp.eCJEPlm3sA
++++ rm /tmp/tmp.WwnFomIb7M /tmp/tmp.eCJEPlm3sA
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.WfzqsCbBsH
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MQGVLuXuA1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.WfzqsCbBsH
+++ cat /tmp/tmp.MQGVLuXuA1
+++ rm /tmp/tmp.WfzqsCbBsH /tmp/tmp.MQGVLuXuA1
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name.recreate-1535
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MDh7bUKzRr
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DgZBPFkpjf
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.MDh7bUKzRr
++ cat /tmp/tmp.DgZBPFkpjf
++ rm /tmp/tmp.MDh7bUKzRr /tmp/tmp.DgZBPFkpjf
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CHvgU4PPkR
++ mktemp
+ local LAST_ERR=/tmp/tmp.9gteLeW1OS
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.CHvgU4PPkR
+ cat /tmp/tmp.9gteLeW1OS
+ rm /tmp/tmp.CHvgU4PPkR /tmp/tmp.9gteLeW1OS
+ return 0
++ seq 1 3
+ for r in '$(seq 1 3)'
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.cYNpQpVwwC
++ mktemp
+ local LAST_ERR=/tmp/tmp.4rgrd4SoKt
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 1}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.cYNpQpVwwC
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.4rgrd4SoKt
+ rm /tmp/tmp.cYNpQpVwwC /tmp/tmp.4rgrd4SoKt
+ return 0
+ sleep 25
+ wait_deployment some-name-repl1
+ local name=some-name-repl1
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl1
+ sleep 35
+ check_replica some-name-repl1 some-name select-2
+ local name=some-name-repl1
+ local cluster=some-name
+ local sql_cmp=select-2
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.3O6DjC7Owh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.o1u2c75LSt
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.3O6DjC7Owh
+++ cat /tmp/tmp.o1u2c75LSt
+++ rm /tmp/tmp.3O6DjC7Owh /tmp/tmp.o1u2c75LSt
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jGC9xHu39R
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MbQ7zoWyah
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.jGC9xHu39R
+++ cat /tmp/tmp.MbQ7zoWyah
+++ rm /tmp/tmp.jGC9xHu39R /tmp/tmp.MbQ7zoWyah
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yg0yFBRa5g
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.SNz8MH0F2W
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.yg0yFBRa5g
+++ cat /tmp/tmp.SNz8MH0F2W
+++ rm /tmp/tmp.yg0yFBRa5g /tmp/tmp.SNz8MH0F2W
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.cdsfPruF2i
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.2t8JTROy1u
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.cdsfPruF2i
++++ cat /tmp/tmp.2t8JTROy1u
++++ rm /tmp/tmp.cdsfPruF2i /tmp/tmp.2t8JTROy1u
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.krlpd6U9DA
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.oPHkEzb3a1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.krlpd6U9DA
+++ cat /tmp/tmp.oPHkEzb3a1
+++ rm /tmp/tmp.krlpd6U9DA /tmp/tmp.oPHkEzb3a1
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nJjKi9cGcj
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DtDTMnvnr7
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.nJjKi9cGcj
++ cat /tmp/tmp.DtDTMnvnr7
++ rm /tmp/tmp.nJjKi9cGcj /tmp/tmp.DtDTMnvnr7
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.J2TOIwKwLl
++ mktemp
+ local LAST_ERR=/tmp/tmp.ciyw9LPbbF
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.J2TOIwKwLl
+ cat /tmp/tmp.ciyw9LPbbF
+ rm /tmp/tmp.J2TOIwKwLl /tmp/tmp.ciyw9LPbbF
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Ve4KEWZFCA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ecawm0FwmM
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Ve4KEWZFCA
++ cat /tmp/tmp.Ecawm0FwmM
++ rm /tmp/tmp.Ve4KEWZFCA /tmp/tmp.Ecawm0FwmM
++ return 0
+ replica_pod_name=some-name-repl1-7987df9b7-2tblz
++ grep some-name-repl1-7987df9b7-2tblz /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl1-7987df9b7-2tblz | 10.55.248.32 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LEEtV0wv0H
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3BHj293TgO
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl1,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.LEEtV0wv0H
++ cat /tmp/tmp.3BHj293TgO
++ rm /tmp/tmp.LEEtV0wv0H /tmp/tmp.3BHj293TgO
++ return 0
+ replica_pod_IP=10.55.248.32
+ compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.248.32 '' ' '
+ local command_id=select-2
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.248.32
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.248.32 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.248.32
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.c7LnQwELaN
+++ mktemp
++ local LAST_ERR=/tmp/tmp.jberNh10SL
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.c7LnQwELaN
++ cat /tmp/tmp.jberNh10SL
++ rm /tmp/tmp.c7LnQwELaN /tmp/tmp.jberNh10SL
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.248.32 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.PRsfdsT6aP
++ mktemp
+ local LAST_ERR=/tmp/tmp.9Ph4CznsIE
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.248.32 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.PRsfdsT6aP
+ cat /tmp/tmp.9Ph4CznsIE
+ rm /tmp/tmp.PRsfdsT6aP /tmp/tmp.9Ph4CznsIE
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.u8o2J7yvV1/select-2.sql
+ for r in '$(seq 1 3)'
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.JAAoZ7i4Dj
++ mktemp
+ local LAST_ERR=/tmp/tmp.7JeS3BVMt4
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 2}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.JAAoZ7i4Dj
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.7JeS3BVMt4
+ rm /tmp/tmp.JAAoZ7i4Dj /tmp/tmp.7JeS3BVMt4
+ return 0
+ sleep 25
+ wait_deployment some-name-repl2
+ local name=some-name-repl2
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl2
+ sleep 35
+ check_replica some-name-repl2 some-name select-2
+ local name=some-name-repl2
+ local cluster=some-name
+ local sql_cmp=select-2
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.rMLu0NVh56
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.8U6g8kqsVs
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.rMLu0NVh56
+++ cat /tmp/tmp.8U6g8kqsVs
+++ rm /tmp/tmp.rMLu0NVh56 /tmp/tmp.8U6g8kqsVs
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ofCYozNdTK
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.kJeHywqMl5
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ofCYozNdTK
+++ cat /tmp/tmp.kJeHywqMl5
+++ rm /tmp/tmp.ofCYozNdTK /tmp/tmp.kJeHywqMl5
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LYr6SuQC1F
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ZKASMp95ID
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.LYr6SuQC1F
+++ cat /tmp/tmp.ZKASMp95ID
+++ rm /tmp/tmp.LYr6SuQC1F /tmp/tmp.ZKASMp95ID
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.nTaWdLcw7R
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.yAT9tqpESS
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.nTaWdLcw7R
++++ cat /tmp/tmp.yAT9tqpESS
++++ rm /tmp/tmp.nTaWdLcw7R /tmp/tmp.yAT9tqpESS
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.QK2x3cx5Qb
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Jcr0jGkKC3
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.QK2x3cx5Qb
+++ cat /tmp/tmp.Jcr0jGkKC3
+++ rm /tmp/tmp.QK2x3cx5Qb /tmp/tmp.Jcr0jGkKC3
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7hAPtdEaMd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.OvJ4DtcFxr
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.7hAPtdEaMd
++ cat /tmp/tmp.OvJ4DtcFxr
++ rm /tmp/tmp.7hAPtdEaMd /tmp/tmp.OvJ4DtcFxr
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.XUvsqggQJe
++ mktemp
+ local LAST_ERR=/tmp/tmp.V34CYu6cSJ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.XUvsqggQJe
+ cat /tmp/tmp.V34CYu6cSJ
+ rm /tmp/tmp.XUvsqggQJe /tmp/tmp.V34CYu6cSJ
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hFiwU6koFS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9LvUq0vQ3y
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.hFiwU6koFS
++ cat /tmp/tmp.9LvUq0vQ3y
++ rm /tmp/tmp.hFiwU6koFS /tmp/tmp.9LvUq0vQ3y
++ return 0
+ replica_pod_name=some-name-repl2-57cf7d6dcf-gghhq
++ grep some-name-repl2-57cf7d6dcf-gghhq /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl2-57cf7d6dcf-gghhq | 10.55.250.70 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.fa9PjXw374
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4aokqcsPtK
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl2,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.fa9PjXw374
++ cat /tmp/tmp.4aokqcsPtK
++ rm /tmp/tmp.fa9PjXw374 /tmp/tmp.4aokqcsPtK
++ return 0
+ replica_pod_IP=10.55.250.70
+ compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.250.70 '' ' '
+ local command_id=select-2
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.250.70
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.250.70 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.250.70
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.9WagSdQ1HD
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6VhFuHJ6vu
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.9WagSdQ1HD
++ cat /tmp/tmp.6VhFuHJ6vu
++ rm /tmp/tmp.9WagSdQ1HD /tmp/tmp.6VhFuHJ6vu
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.250.70 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.uE4oa6yXWK
++ mktemp
+ local LAST_ERR=/tmp/tmp.A5F0zUMHLm
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.250.70 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.uE4oa6yXWK
+ cat /tmp/tmp.A5F0zUMHLm
+ rm /tmp/tmp.uE4oa6yXWK /tmp/tmp.A5F0zUMHLm
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.u8o2J7yvV1/select-2.sql
+ for r in '$(seq 1 3)'
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Gyge60XccO
++ mktemp
+ local LAST_ERR=/tmp/tmp.typD0Vos3M
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op": "replace", "path": "/spec/pgReplicas/hotStandby/size", "value": 3}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Gyge60XccO
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.typD0Vos3M
+ rm /tmp/tmp.Gyge60XccO /tmp/tmp.typD0Vos3M
+ return 0
+ sleep 25
+ wait_deployment some-name-repl3
+ local name=some-name-repl3
+ local target_namespace=recreate-1535
+ sleep 10
+ set +o xtrace
some-name-repl3
+ sleep 35
+ check_replica some-name-repl3 some-name select-2
+ local name=some-name-repl3
+ local cluster=some-name
+ local sql_cmp=select-2
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=recreate-1535
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ssHGK9yeOz
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VqmtYDH1Ex
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ssHGK9yeOz
+++ cat /tmp/tmp.VqmtYDH1Ex
+++ rm /tmp/tmp.ssHGK9yeOz /tmp/tmp.VqmtYDH1Ex
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.kj8x0DRQ63
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S465QBBFT3
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.kj8x0DRQ63
+++ cat /tmp/tmp.S465QBBFT3
+++ rm /tmp/tmp.kj8x0DRQ63 /tmp/tmp.S465QBBFT3
+++ return 0
+++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JfZA2mNkmg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.IyUuSZ6ujL
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.JfZA2mNkmg
+++ cat /tmp/tmp.IyUuSZ6ujL
+++ rm /tmp/tmp.JfZA2mNkmg /tmp/tmp.IyUuSZ6ujL
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.vLL4uJvOH4
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.e7F6NrHuxR
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n recreate-1535 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.vLL4uJvOH4
++++ cat /tmp/tmp.e7F6NrHuxR
++++ rm /tmp/tmp.vLL4uJvOH4 /tmp/tmp.e7F6NrHuxR
++++ return 0
+++ kubectl_bin -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zjxpA5k5xE
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.B0wrJguIbF
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n recreate-1535 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.zjxpA5k5xE
+++ cat /tmp/tmp.B0wrJguIbF
+++ rm /tmp/tmp.zjxpA5k5xE /tmp/tmp.B0wrJguIbF
+++ return 0
++ echo postgres_password
+ run_psql 'SELECT usename,application_name,client_addr,state from pg_stat_replication' postgres:postgres_password@some-name.recreate-1535
+ local 'command=SELECT usename,application_name,client_addr,state from pg_stat_replication'
+ local uri=postgres:postgres_password@some-name.recreate-1535
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ZMlcsfzEGe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.RT1pQgm6z5
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.ZMlcsfzEGe
++ cat /tmp/tmp.RT1pQgm6z5
++ rm /tmp/tmp.ZMlcsfzEGe /tmp/tmp.RT1pQgm6z5
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.NlxElV7P1h
++ mktemp
+ local LAST_ERR=/tmp/tmp.6SKIt06iw6
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''SELECT usename,application_name,client_addr,state from pg_stat_replication\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.recreate-1535.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.NlxElV7P1h
+ cat /tmp/tmp.6SKIt06iw6
+ rm /tmp/tmp.NlxElV7P1h /tmp/tmp.6SKIt06iw6
+ return 0
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.okvIcPtbKJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.O5V7ZGSR5Z
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.okvIcPtbKJ
++ cat /tmp/tmp.O5V7ZGSR5Z
++ rm /tmp/tmp.okvIcPtbKJ /tmp/tmp.O5V7ZGSR5Z
++ return 0
+ replica_pod_name=some-name-repl3-97b8b7b55-hs72z
++ grep some-name-repl3-97b8b7b55-hs72z /tmp/tmp.u8o2J7yvV1/replicas.list
++ grep streaming
+ [[ -z  primaryuser | some-name-repl3-97b8b7b55-hs72z  | 10.55.249.15 | streaming ]]
++ kubectl_bin -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.iJ3xKWajeV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.93PDvZwOSf
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=deployment-name=some-name-repl3,name=some-name-replica -o 'jsonpath={.items[0].status.podIP}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.iJ3xKWajeV
++ cat /tmp/tmp.93PDvZwOSf
++ rm /tmp/tmp.iJ3xKWajeV /tmp/tmp.93PDvZwOSf
++ return 0
+ replica_pod_IP=10.55.249.15
+ compare_psql_cmd select-2 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.249.15 '' ' '
+ local command_id=select-2
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.249.15
+ local postfix=
+ local 'uri_suffix= '
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@10.55.249.15 postgres ' '
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@10.55.249.15
+ local driver=postgres
+ local 'suffix= '
++ kubectl_bin -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.AcJ4RduDmB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rvBdbsRv8t
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n recreate-1535 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.AcJ4RduDmB
++ cat /tmp/tmp.rvBdbsRv8t
++ rm /tmp/tmp.AcJ4RduDmB /tmp/tmp.rvBdbsRv8t
++ return 0
+ local client_container=pg-client-76df9df876-jkgv9
+ kubectl_bin exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.249.15 '
++ mktemp
+ local LAST_OUT=/tmp/tmp.ldHCCqDWQu
++ mktemp
+ local LAST_ERR=/tmp/tmp.YrmkHfQxDK
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-jkgv9 -n recreate-1535 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@10.55.249.15 '
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ldHCCqDWQu
+ cat /tmp/tmp.YrmkHfQxDK
+ rm /tmp/tmp.ldHCCqDWQu /tmp/tmp.YrmkHfQxDK
+ return 0
+ '[' '!' -s /tmp/tmp.u8o2J7yvV1/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.u8o2J7yvV1/select-2.sql
++ kubectl_bin get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MrEuWIFz59
+++ mktemp
++ local LAST_ERR=/tmp/tmp.1u6qAWtCnH
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=role=master,service-name=some-name,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.MrEuWIFz59
++ cat /tmp/tmp.1u6qAWtCnH
++ rm /tmp/tmp.MrEuWIFz59 /tmp/tmp.1u6qAWtCnH
++ return 0
+ master_container=some-name-696f799f57-lr8xh
+ kubectl_bin get configmap/some-name-pgha-config -o yaml
+ yq w - postgresql.parameters.jit false
+ yq r - 'data[some-name-dcs-config]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.UY6GThNnxy
++ mktemp
+ local LAST_ERR=/tmp/tmp.ikF00yMlAX
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get configmap/some-name-pgha-config -o yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.UY6GThNnxy
+ cat /tmp/tmp.ikF00yMlAX
+ rm /tmp/tmp.UY6GThNnxy /tmp/tmp.ikF00yMlAX
+ return 0
+ kubectl_bin apply -f -
+ kubectl_bin get configmap/some-name-pgha-config -o yaml
++ mktemp
++ mktemp
++ cat /tmp/tmp.u8o2J7yvV1/some-name-pgha-config.patch
+ local LAST_OUT=/tmp/tmp.chf4CED85g
+ local LAST_OUT=/tmp/tmp.IIu0dWL7tK
++ mktemp
++ mktemp
+ yq w - 'data[some-name-dcs-config]' 'postgresql:
  parameters:
    jit: false
    unix_socket_directories: /tmp
    wal_level: logical
    archive_mode: true
    archive_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-push "%p"
    archive_timeout: 60
    log_directory: pg_log
    shared_buffers: 128MB
    temp_buffers: 8MB
    log_min_duration_statement: 60000
    log_statement: none
    work_mem: 4MB
    max_wal_senders: 6
    shared_preload_libraries: pg_stat_monitor,pgaudit
    ssl: "on"
    ssl_cert_file: /pgconf/tls/tls.crt
    ssl_key_file: /pgconf/tls/tls.key
    ssl_ca_file: /pgconf/tls/ca.crt
  use_slots: false
  recovery_conf:
    restore_command: source /opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-set-env.sh && pgbackrest archive-get %f "%p"
  use_pg_rewind: true'
+ local LAST_ERR=/tmp/tmp.ILU4OrA1Xr
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ local LAST_ERR=/tmp/tmp.qBy80qaxHM
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ seq 0 2
+ kubectl apply -f -
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get configmap/some-name-pgha-config -o yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.chf4CED85g
+ cat /tmp/tmp.ILU4OrA1Xr
+ rm /tmp/tmp.chf4CED85g /tmp/tmp.ILU4OrA1Xr
+ return 0
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.IIu0dWL7tK
configmap/some-name-pgha-config configured
+ cat /tmp/tmp.qBy80qaxHM
Warning: resource configmaps/some-name-pgha-config is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.IIu0dWL7tK /tmp/tmp.qBy80qaxHM
+ return 0
++ kubectl_bin get configmap/some-name-pgha-config -o yaml
++ yq r - 'data[some-name-dcs-config]'
++ yq r - postgresql.parameters.jit
+++ mktemp
++ local LAST_OUT=/tmp/tmp.uevuucUWz7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hjeyVxw0jG
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get configmap/some-name-pgha-config -o yaml
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.uevuucUWz7
++ cat /tmp/tmp.hjeyVxw0jG
++ rm /tmp/tmp.uevuucUWz7 /tmp/tmp.hjeyVxw0jG
++ return 0
+ CONF_MAP_JIT=false
++ kubectl_bin exec some-name-696f799f57-lr8xh -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" '
+++ mktemp
++ local LAST_OUT=/tmp/tmp.NbWnQyNEgM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.oBrDJhyfL3
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl exec some-name-696f799f57-lr8xh -- bash -c 'patronictl show-config | grep jit | grep -Eo "true|false" '
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.NbWnQyNEgM
++ cat /tmp/tmp.oBrDJhyfL3
++ rm /tmp/tmp.NbWnQyNEgM /tmp/tmp.oBrDJhyfL3
++ return 0
+ PATRONI_JIT=false
+ [[ false != false ]]
+ destroy recreate-1535
+ grep -v level=debug
++ get_operator_pod
+ grep -v 'Getting tasks for pod'
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+ grep -v 'the object has been modified'
+ grep -v 'Getting pods from source'
+++ mktemp
+ tee /tmp/tmp.u8o2J7yvV1/operator.log
+ sort -u
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ grep -v level=info
++ local LAST_OUT=/tmp/tmp.gTjOwG3P5t
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Due5vboIdY
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.gTjOwG3P5t
++ cat /tmp/tmp.Due5vboIdY
++ rm /tmp/tmp.gTjOwG3P5t /tmp/tmp.Due5vboIdY
++ return 0
+ kubectl_bin logs postgres-operator-56696f7849-zfjp8 -c operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.qTVkYDAGnV
++ mktemp
+ local LAST_ERR=/tmp/tmp.M6qaU7C6e4
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl logs postgres-operator-56696f7849-zfjp8 -c operator
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.qTVkYDAGnV
+ cat /tmp/tmp.M6qaU7C6e4
+ rm /tmp/tmp.qTVkYDAGnV /tmp/tmp.M6qaU7C6e4
+ return 0

  
    
        
            
                
                  
                    
                      
                        
                          
                            
                                
                                        
                                            
                                            ]
                                        {
                                        }
                                        },
                                    ]
                                    }
                                    },
                                  {
                                  }
                                  },
                                ]
                                {
                                }
                                },
                              {
                              }
                            ]
                            }
                          {
                          }
                          },
                         ]
                         }
                        ]
                        {
                        }
                        },
                        }, {
                      ]
                      {
                      }
                      },
                     ]
                    ]
                    ],
                    {
                    }
                    },
                    },  {
                    }, {
                    },{
                    }]
                    }],
                  ]
                  {
                  }
                  },
                ,
                ],
                {
                }
                },
                }],
            {
            }
            },
            }],
          }
        ],
        }
        },
    }
    },
  }
 },
{
}
},
	  },
		
2024/04/23 15:45:17 INF    1 (localhost:4150) connecting to nsqd
2024/04/23 15:46:06 INF    2 (localhost:4150) connecting to nsqd
2024/04/23 15:46:07 INF    3 (localhost:4150) connecting to nsqd
2024/04/23 15:46:13 INF    4 (localhost:4150) connecting to nsqd
2024/04/23 15:46:41 INF    5 (localhost:4150) connecting to nsqd
2024/04/23 15:46:41 INF    6 (localhost:4150) connecting to nsqd
2024/04/23 15:46:41 INF    7 (localhost:4150) connecting to nsqd
2024/04/23 15:47:33 INF    8 (localhost:4150) connecting to nsqd
2024/04/23 15:47:52 INF    9 (localhost:4150) connecting to nsqd
2024/04/23 15:49:02 INF   10 (localhost:4150) connecting to nsqd
2024/04/23 15:50:22 INF   11 (localhost:4150) connecting to nsqd
2024/04/23 15:51:41 INF   12 (localhost:4150) connecting to nsqd
2024/04/23 15:56:53 INF   13 (localhost:4150) connecting to nsqd
2024/04/23 15:57:42 INF   14 (localhost:4150) connecting to nsqd
2024/04/23 15:58:27 INF   15 (localhost:4150) connecting to nsqd
2024/04/23 15:59:12 INF   16 (localhost:4150) connecting to nsqd
2024/04/23 15:59:29 INF   17 (localhost:4150) connecting to nsqd
2024/04/23 15:59:34 INF   18 (localhost:4150) connecting to nsqd
2024/04/23 16:00:28 INF   19 (localhost:4150) connecting to nsqd
2024/04/23 16:01:24 INF   20 (localhost:4150) connecting to nsqd
2024/04/23 16:01:24 INF   21 (localhost:4150) connecting to nsqd
2024/04/23 16:01:31 INF   22 (localhost:4150) connecting to nsqd
2024/04/23 16:01:50 INF   23 (localhost:4150) connecting to nsqd
2024/04/23 16:01:50 INF   24 (localhost:4150) connecting to nsqd
2024/04/23 16:01:51 INF   25 (localhost:4150) connecting to nsqd
2024/04/23 16:02:31 INF   26 (localhost:4150) connecting to nsqd
2024/04/23 16:03:51 INF   27 (localhost:4150) connecting to nsqd
2024/04/23 16:05:11 INF   28 (localhost:4150) connecting to nsqd
        "accessModes": [
                "affinity": 
                "affinity": {
                "affinity": {                 "podAntiAffinity": {
                      "allowPrivilegeEscalation": false,
                "annotations": {"keep-backups":"false","keep-data":"false"},
    "apiVersion": "apps/v1",
    "apiVersion": "batch/v1",
                                        "apiVersion": "v1",
        "apiVersion": "v1",
    "apiVersion": "v1",
                    "backrest-command": "backup",
                    "backrest-command": "stanza-create",
                        "claimName": "some-name-pgbr-repo"
                            "command": [
                                    "configMap": {
                          "configMap": {
                            "configMapKeyRef": {
                                        "containerName": "database",
                        "containerPort": 2022,
                        "containerPort": 5432,
                        "containerPort": 8009,
                "containers": [
                "containers": [{
    "cpu": "1",
    "cpu": "200m",
    "cpu": "500m",
                                                "crunchydata"
                    "crunchy-pgbouncer": "true",
                "crunchy-pgbouncer": "true",
            "crunchy-pgbouncer": "true",
                            "defaultMode": 288
                        "defaultMode": 288,
                    "defaultMode": 288,
                            "defaultMode": 420,
            "deployment-name": "some-name"
                    "deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name"
            "deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name"
            "deployment-name": "some-name-repl1"
            "deployment-name": "some-name-repl2"
            "deployment-name": "some-name-repl3"
            "deployment-name": "some-name-repl3","pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name"
                                        "divisor": "1m",
                "dnsPolicy": "ClusterFirst"
                        "downwardAPI": {
                        "emptyDir": {
                      "emptyDir": {
                    "emptyDir": {
                  "emptyDir": {
                    "env": [
                    "env": [{
                        "exec": {
                                        "fieldPath": "metadata.annotations"
                                        "fieldPath": "metadata.labels"
                                "fieldPath": "metadata.labels['crunchy-pgha-scope']"
                                "fieldPath": "metadata.namespace"
                                    "fieldRef": {
                            "fieldRef": {
                  "fsGroup": 2,
I0423 15:44:42.337012       1 shared_informer.go:240] Waiting for caches to sync for namespace
I0423 15:44:42.437455       1 shared_informer.go:247] Caches are synced for namespace 
I0423 15:44:42.575345       1 shared_informer.go:240] Waiting for caches to sync for recreate-1535
I0423 15:44:42.676502       1 shared_informer.go:247] Caches are synced for recreate-1535 
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha",
                    "image": "perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-rmdata",
                    "imagePullPolicy": "IfNotPresent"
                    "imagePullPolicy": "IfNotPresent",
                        "initialDelaySeconds": 15
                        "initialDelaySeconds": 30,
                                "items": [
                            "items": [
                      "items": [
                                 "key": "config",
                           "key": "config",
                                            "key": "crunchy-pgbouncer",
                               "key": "init"
                                "key": "password"
                                            "key": "pg-cluster",
                                            "key": "pgo-backrest",
                                            "key": "pgo-backrest-job",
                                            "key": "pgo-pg-database",
                                            "key": "pg-pod-anti-affinity",
                                    "key": "tls.crt",
                                    "key": "tls.key",
                                            "key": "vendor",
    "kind": "Deployment",
    "kind": "Job",
    "kind": "PersistentVolumeClaim",
        "kind": "Service",
                "labels": {
            "labels": {
        "labels": {
                                "labelSelector": {
                    "livenessProbe": {
                                    "matchExpressions": [
            "matchLabels": {
          "matchLabels": {
                "maxSurge": 1
                "maxUnavailable": 1,
                          "medium": "Memory",
                        "medium": "Memory"
                        "medium": "Memory",
                      "medium": "Memory",
                    "medium": "Memory",
    "memory": "128Mi"
    "memory": "256Mi"
    "memory": "48Mi"
            "metadata": {
        "metadata": {
    "metadata": {
                      "mountPath": "/backrestrepo",
                            "mountPath": "/dev/shm",
                      "mountPath": "/etc/pgbackrest"
                            "mountPath": "/etc/pgbackrest/conf.d",
                            "mountPath": "/etc/podinfo",
                          "mountPath": "/etc/ssh",
                      "mountPath": "/etc/ssh",
                      "mountPath": "/home/pgbackrest"
                            "mountPath": "/pgconf",
                        "mountPath": "/pgconf/",
                            "mountPath": "/pgconf/pgreplicator",
                            "mountPath": "/pgconf/pgsuper",
                            "mountPath": "/pgconf/pguser",
                          "mountPath": "/pgconf/tls",
                        "mountPath": "/pgconf/tls/pgbouncer",
                            "mountPath": "/pgconf/tls-replication",
                            "mountPath": "/pgdata",
                            "mountPath": "/sshd",
                      "mountPath": "/sshd",
                            "mountPath": "/tmp",
                          "mountPath": "/tmp",
                      "mountPath": "/tmp"
                    "name": "backrest",
                "name": "backrest-backup-some-name",
                      "name": "backrestrepo",
                    "name": "backrestrepo",
  "name": "BACKREST_SKIP_CREATE_STANZA",
                                "name": "cluster1-replication-ssl-keypair",
                                "name": "cluster1-ssl-ca"
                              "name": "cluster1-ssl-ca"
                                "name": "cluster1-ssl-keypair"
                              "name": "cluster1-ssl-keypair"
                        "name": "COMMAND",
                        "name": "COMMAND_OPTS",
                    "name": "database",
                            "name": "dshm"
                      "name": "dshm",
  "name": "ENABLE_SSHD",
                        "name": "IS_BACKUP",
                        "name": "IS_REPLICA",
                        "name": "LD_PRELOAD",
                      "name": "LD_PRELOAD",
                        "name": "MODE",
                        "name": "NAMESPACE",
                        "name": "NSS_WRAPPER_GROUP",
                      "name": "NSS_WRAPPER_GROUP",
                        "name": "NSS_WRAPPER_PASSWD",
                      "name": "NSS_WRAPPER_PASSWD",
                        "name": "patroni"
                        "name": "PATRONI_KUBERNETES_LABELS",
                        "name": "PATRONI_KUBERNETES_NAMESPACE",
                        "name": "PATRONI_KUBERNETES_SCOPE_LABEL",
                        "name": "PATRONI_LOG_LEVEL",
                        "name": "PATRONI_POSTGRESQL_DATA_DIR",
                        "name": "PATRONI_SCOPE",
                        "name": "pgbackrest"
                      "name": "pgbackrest-conf",
                  "name": "pgbackrest-conf",
                            "name": "pgbackrest-config"
                      "name": "pgbackrest-config",
                        "name": "PGBACKREST_DB_HOST",
                        "name": "PGBACKREST_DB_PATH",
  "name": "PGBACKREST_DB_PATH",
                      "name": "pgbackrest-home",
                  "name": "pgbackrest-home",
                        "name": "PGBACKREST_LOG_PATH",
  "name": "PGBACKREST_LOG_PATH",
                        "name": "PGBACKREST_PG1_PORT",
  "name": "PGBACKREST_PG1_PORT",
                        "name": "PGBACKREST_PG1_SOCKET_PATH",
  "name": "PGBACKREST_PG1_SOCKET_PATH",
  "name": "PGBACKREST_REPO1_HOST",
                        "name": "PGBACKREST_REPO1_PATH",
  "name": "PGBACKREST_REPO1_PATH",
                        "name": "PGBACKREST_REPO1_TYPE",
  "name": "PGBACKREST_REPO1_TYPE",
                        "name": "PGBACKREST_STANZA",
  "name": "PGBACKREST_STANZA",
                        "name": "pgbouncer"
                    "name": "pgbouncer",
                        "name": "pgbouncer-conf",
                    "name": "pgbouncer-conf",
                        "name": "PG_CLUSTER",
                            "name": "pgconf-volume"
                        "name": "pgconf-volume",
                            "name": "pgdata",
                        "name": "pgdata",
                        "name": "PGHA_DATABASE",
                        "name": "PGHA_INIT",
                        "name": "PGHA_PASSWORD_TYPE",
  "name": "PGHA_PGBACKREST",
                        "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
  "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
                        "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
  "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
                        "name": "PGHA_PGBACKREST_S3_VERIFY_TLS",
                        "name": "PGHA_PG_PORT",
                        "name": "PGHA_REPLICA_REINIT_ON_START_FAIL",
                        "name": "PGHA_SCOPE",
                        "name": "PGHA_STANDBY",
                        "name": "PGHA_SYNC_REPLICATION",
                        "name": "PGHA_TLS_ENABLED",
                        "name": "PGHA_TLS_ONLY",
                        "name": "PGHA_USER",
                        "name": "PGHOST",
                        "name": "PG_PASSWORD",
                        "name": "PG_PRIMARY_SERVICE_NAME",
                        "name": "PITR_TARGET",
                            "name": "podinfo"
                        "name": "podinfo",
                        "name": "PODNAME",
                        "name": "postgres"
                            "name": "primary-volume"
                        "name": "primary-volume",
                        "name": "REMOVE_BACKUP",
                        "name": "REMOVE_DATA",
                        "name": "REPLICA_NAME",
                        "name": "report",
                    "name": "rmdata",
                            "name": "root-volume"
                        "name": "root-volume",
                    "name": "some-name",
        "name": "some-name",
                    "name": "some-name-backrest-shared-repo",
                "name": "some-name-backrest-shared-repo"
                "name": "some-name-backrest-shared-repo",
            "name": "some-name-backrest-shared-repo",
        "name": "some-name-backrest-shared-repo",
                    "name": "some-name-pgbouncer",
                "name": "some-name-pgbouncer",
            "name": "some-name-pgbouncer",
        "name": "some-name-pgbouncer",
                            "name": "some-name-pgbouncer-cm"
                                "name": "some-name-pgbouncer-secret",
                            "name": "some-name-pgbouncer-secret",
        "name": "some-name-pgbr-repo",
                    "name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0"
                                        "name": "some-name-pgha-config",
                               "name": "some-name-pgha-config",
                    "name": "some-name-repl1",
        "name": "some-name-repl1",
                    "name": "some-name-repl2",
        "name": "some-name-repl2",
                    "name": "some-name-repl3",
        "name": "some-name-repl3",
            "name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1","pgo-version": "1.6.0"
            "name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0"
                "name": "some-name-rmdata-wfrg",
        "name": "some-name-rmdata-wfrg",
                "name": "some-name-stanza-create",
                          "name": "ssh-config",
                        "name": "ssh-config",
                      "name": "ssh-config",
                  "name": "ssh-config",
                            "name": "sshd",
                        "name": "sshd",
                      "name": "sshd",
                    "name": "sshd",
                        "name": "tls-pgbouncer"
                    "name": "tls-pgbouncer",
                            "name": "tls-replication"
                        "name": "tls-replication",
                          "name": "tls-server"
                      "name": "tls-server",
                            "name": "tmp"
                          "name": "tmp"
                      "name": "tmp",
                    "name": "tmp",
                  "name": "tmp",
                            "name": "user-volume"
                        "name": "user-volume",
                "nodePort": 0
                                            "operator": "Exists"
                                            "operator": "In",
                                "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh"
                                "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh"
                                        "optional": true
                                    "path": "annotations"
                                    "path": "cpu_limit",
                                    "path": "cpu_request",
                                    "path": "labels"
                                    "path": "mem_limit",
                                    "path": "mem_request",
                                 "path": "ssh_config"
                           "path": "ssh_config"
                                    "path": "tls-replication.crt"
                                    "path": "tls-replication.key"
                        "periodSeconds": 15,
                    "persistentVolumeClaim": {
                        "persistentVolumeClaim":{"claimName":"some-name"}
                        "persistentVolumeClaim":{"claimName":"some-name-repl1"}
                        "persistentVolumeClaim":{"claimName":"some-name-repl2"}
                        "persistentVolumeClaim":{"claimName":"some-name-repl3"}
                    "pg-cluster": "some-name"
                    "pg-cluster": "some-name",
                "pg-cluster": "some-name"
                "pg-cluster": "some-name",
            "pg-cluster": "some-name"
            "pg-cluster": "some-name",
            "pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name"
                    "pg-cluster": "some-name","deployment-name": "some-name-repl1","pgo-version": "1.6.0","name": "some-name-replica"
                    "pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0","name": "some-name-replica"
            "pg-cluster": "some-name","deployment-name": "some-name-repl2","pgo-version": "1.6.0","name": "some-name-replica"
                    "pgo-backrest-job": "true",
                    "pgo-backrest-repo": "true"
                "pgo-backrest-repo": "true",
            "pgo-backrest-repo": "true"
                    "pgo-backrest": "true",
                    "pgo-pg-database": "true",
            "pgo-pg-database": "true",
                    "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1"
            "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1"
                    "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2"
            "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2"
                    "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3"
            "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl3"
                    "pg-pod-anti-affinity": "preferred",
                "pg-pod-anti-affinity": "preferred",
                    "pgrmdata": "true",
            "pgrmdata": "true",
                            "podAffinityTerm": {
                                        "podAntiAffinity": {
                "podAntiAffinity": {
		                "podAntiAffinity": {
                "port": 2022,
                    "ports": [{
            "ports": [{
                    "preferredDuringSchedulingIgnoredDuringExecution": [
                      "privileged": false,
                        "projected": {
                      "projected": {
                    "projected": {
                      "projected": { "sources": [] }
                        "protocol": "TCP",
                "protocol": "TCP",
                    "readinessProbe": {
                            "readOnly": false
                        "readOnly": false
                      "readOnly": false
                      "readOnlyRootFilesystem": true
                            "readOnly": true
                          "readOnly": true
                      "readOnly": true
            "ReadWriteOnce"
        "replicas": 0,
        "replicas": 1,
        "replicas": 3,
            "requests": {
  "requests": {
                                    "resourceFieldRef": {
                                        "resource": "limits.cpu"
                                        "resource": "limits.memory"
                                        "resource": "requests.cpu"
                                        "resource": "requests.memory"
        "resources": {
"resources": {
                "restartPolicy": "Always",
                "restartPolicy": "Never"
            "rollingUpdate": {
                  "runAsNonRoot": true
                            "secret": {
                          "secret": {
                        "secret": {
                    "secret": {
                  "secret": {
                            "secretKeyRef": {
                            "secretName": "some-name-backrest-repo-config"
                            "secretName": "some-name-backrest-repo-config",
                        "secretName": "some-name-backrest-repo-config"
                      "secretName": "some-name-backrest-repo-config",
                            "secretName": "some-name-postgres-secret"
                            "secretName": "some-name-primaryuser-secret"
                            "secretName": "some-name-some-name-secret"
                    "securityContext": {
                "securityContext": {
                "securityContext": {},
                "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]},
                "securityContext": {"runAsNonRoot":true,"fsGroup":26},
            "selector": {
        "selector": {
                "serviceAccountName": "pgo-backrest",
                "serviceAccountName": "pgo-default",
                "serviceAccountName": "pgo-pg",
                "serviceAccountName": "pgo-target",
                    "service-name": "some-name-backrest-shared-repo",
            "service-name": "some-name-backrest-shared-repo",
                    "service-name": "some-name-pgbouncer",
                "service-name": "some-name-pgbouncer",
            "service-name": "some-name-pgbouncer",
            "sessionAffinity": "None"
                    "sizeLimit": "128Ki"
                        "sizeLimit": "16Mi"
                      "sizeLimit": "1Mi"
                          "sizeLimit": "2Mi"
                          "sizeLimit": "64Mi"
                    "sizeLimit": "64Mi"
                                                "some-name"
                            "sources": [
                        "sources": [
                      "sources": [
            "spec": {
        "spec": {
    "spec": {
                "storage": "1Gi"
        "strategy": {
                "targetPort": 2022,
        "template": {
time="2024-04-23T15:44:41Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0
time="2024-04-23T15:45:21Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T15:45:26Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T15:46:06Z" level=error msg="Error completing  workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T15:46:06Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T15:46:41Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T15:46:41Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T15:46:42Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0
time="2024-04-23T15:46:42Z" level=error msg="pod some-name-repl1-7987df9b7-6wxhr does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:46:42Z" level=error msg="pod some-name-repl1-7987df9b7-6wxhr does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T15:46:42Z" level=error msg="pod some-name-repl2-57cf7d6dcf-jv698 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:46:42Z" level=error msg="pod some-name-repl2-57cf7d6dcf-jv698 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T15:46:43Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0
time="2024-04-23T15:46:43Z" level=error msg="pod some-name-repl1-7987df9b7-6wxhr does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:46:43Z" level=error msg="pod some-name-repl1-7987df9b7-6wxhr does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T15:46:43Z" level=error msg="pod some-name-repl2-57cf7d6dcf-jv698 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:46:43Z" level=error msg="pod some-name-repl2-57cf7d6dcf-jv698 does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T15:50:22Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T15:51:41Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T15:51:42Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0
time="2024-04-23T15:51:42Z" level=error msg="pod some-name-repl3-97b8b7b55-jn56s does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:51:42Z" level=error msg="pod some-name-repl3-97b8b7b55-jn56s does not have a host assigned" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T15:53:42Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0
time="2024-04-23T15:53:42Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:53:42Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0
time="2024-04-23T15:55:54Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T15:55:54Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
time="2024-04-23T15:59:12Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:13Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:14Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:15Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:16Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:17Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:21Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:22Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:26Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:28Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:29Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T15:59:34Z" level=error msg="jobs.batch \"some-name-rmdata-wfrg\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T16:00:32Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T16:00:37Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T16:01:24Z" level=error msg="Error completing  workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T16:01:24Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T16:01:51Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T16:01:51Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T16:03:51Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T16:05:11Z" level=warning msg="crv1 pgreplica some-name-repl3 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
                        "timeoutSeconds": 10
                                "topologyKey": "kubernetes.io/hostname"
            "type": "ClusterIP",
            "type": "RollingUpdate",
                        "value": ""
                        "value": "5432"
  "value": "5432"
                        "value": "/backrestrepo/some-name-backrest-shared-repo"
  "value": "/backrestrepo/some-name-backrest-shared-repo"
                        "value": "backup"
                        "value": "crunchy-pgha-scope"
                        "value": "db"
  "value": "db"
                        "value": " --db-host=10.55.249.13 --db-path=/pgdata/some-name"
                        "value": " --db-host=10.55.249.7 --db-path=/pgdata/some-name"
                        "value": "false"
  "value": "false"
                        "valueFrom": {
                        "value": "INFO"
                        "value": "pgbackrest"
                        "value": "pgbackrest-repo"
                        "value": "/pgdata/some-name"
  "value": "/pgdata/some-name"
                        "value": "/pgdata/some-name-repl1"
  "value": "/pgdata/some-name-repl1"
                        "value": "/pgdata/some-name-repl2"
  "value": "/pgdata/some-name-repl2"
                        "value": "/pgdata/some-name-repl3"
  "value": "/pgdata/some-name-repl3"
                        "value": "posix"
  "value": "posix"
                        "value": "postgres"
                                            "values": [
                        "value": "some-name"
  "value": "some-name-backrest-shared-repo"
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-jggm7"
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-zc7rc"
                        "value": "stanza-create"
                        "value": "/tmp"
  "value": "/tmp"
                        "value": "/tmp/nss_wrapper/pgbackrest-repo/group"
                        "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd"
                      "value": "/tmp/nss_wrapper/postgres/group"
                      "value": "/tmp/nss_wrapper/postgres/passwd"
                        "value": "true"
  "value": "true"
                        "value": "--type=full --db-host=10.55.249.13 --db-path=/pgdata/some-name"
                        "value": "--type=full --db-host=10.55.249.7 --db-path=/pgdata/some-name"
                        "value": "/usr/lib64/libnss_wrapper.so"
                      "value": "/usr/lib64/libnss_wrapper.so"
                        "value": "{vendor: \"crunchydata\"}"
                    "vendor": "crunchydata"
                    "vendor": "crunchydata",
                "vendor": "crunchydata"
                "vendor": "crunchydata",
            "vendor": "crunchydata"
            "vendor": "crunchydata",
                    "volumeMounts": [
                    "volumeMounts": [{
                "volumes": [
                "volumes": [{
                            "weight": 1,
+ destroy_operator
+ kubectl_bin delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.bnjPY1xCVl
++ mktemp
+ local LAST_ERR=/tmp/tmp.zldrSPJBRN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.u8o2J7yvV1/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.bnjPY1xCVl
+ cat /tmp/tmp.zldrSPJBRN
Error from server (NotFound): error when deleting "/tmp/tmp.u8o2J7yvV1/operator.yaml": serviceaccounts "pgo-deployer-sa" not found
Error from server (NotFound): error when deleting "/tmp/tmp.u8o2J7yvV1/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): error when deleting "/tmp/tmp.u8o2J7yvV1/operator.yaml": configmaps "pgo-deployer-cm" not found
Error from server (NotFound): error when deleting "/tmp/tmp.u8o2J7yvV1/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found
Error from server (NotFound): error when deleting "/tmp/tmp.u8o2J7yvV1/operator.yaml": jobs.batch "pgo-deploy" not found
+ rm /tmp/tmp.bnjPY1xCVl /tmp/tmp.zldrSPJBRN
+ return 1
+ true
+ yq w -d4 /tmp/tmp.u8o2J7yvV1/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall
+ kubectl_bin apply -f /tmp/tmp.u8o2J7yvV1/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.6DXEA9cs0n
++ mktemp
+ local LAST_ERR=/tmp/tmp.J7PzbZvOPF
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.u8o2J7yvV1/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6DXEA9cs0n
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.J7PzbZvOPF
+ rm /tmp/tmp.6DXEA9cs0n /tmp/tmp.J7PzbZvOPF
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=recreate-1535
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n recreate-1535 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.u8o2J7yvV1/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.isZP7dvJKo
++ mktemp
+ local LAST_ERR=/tmp/tmp.TpHbutlgMU
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.u8o2J7yvV1/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.isZP7dvJKo
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.TpHbutlgMU
+ rm /tmp/tmp.isZP7dvJKo /tmp/tmp.TpHbutlgMU
+ return 0
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns --grace-period=0 --force=true
+ local LAST_OUT=/tmp/tmp.TJgi8XKGMA
++ mktemp
+ local LAST_ERR=/tmp/tmp.FsFgCX9bYY
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get ns
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.TJgi8XKGMA
+ cat /tmp/tmp.FsFgCX9bYY
+ rm /tmp/tmp.TJgi8XKGMA /tmp/tmp.FsFgCX9bYY
+ return 0
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
namespace "recreate-1535" force deleted
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.17ihLikuxQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.kfRf4DeQfK
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.17ihLikuxQ
+ cat /tmp/tmp.kfRf4DeQfK
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.17ihLikuxQ /tmp/tmp.kfRf4DeQfK
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.EDCJzp47NH
++ mktemp
+ local LAST_ERR=/tmp/tmp.75unO5DE4T
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.EDCJzp47NH
+ cat /tmp/tmp.75unO5DE4T
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.EDCJzp47NH /tmp/tmp.75unO5DE4T
+ return 1
+ true
+ rm -rf /tmp/tmp.u8o2J7yvV1