++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/clone-cluster.log'
Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/logs/clone-cluster.log
++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/cloud-secret.yml ']'
++ SKIP_BACKUPS_TO_AWS_GCP=
++ oc get projects
error: the server doesn't have a resource type "projects"
+++ kubectl version -o json
+++ jq -r .serverVersion.gitVersion
+++ grep '\-eks\-'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ '[' ']'
+++ kubectl version -o json
+++ jq -r '.serverVersion.major + "." + .serverVersion.minor'
+++ /usr/bin/sed -r 's/[^0-9.]+//g'
WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1
++ KUBE_VERSION=1.26
+++ helm version -c
+++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/'
++ HELM_VERSION=v3.14.4
++ '[' v3 == v2 ']'
+ main
+ create_namespace clone-cluster-20561
+ local namespace=clone-cluster-20561
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ cleanup_rbac
++ mktemp
+ xargs kubectl delete ns
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ awk '{print$1}'
+ local LAST_OUT=/tmp/tmp.5srTFwL1JJ
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.JTUEY2rNoM
+ local LAST_ERR=/tmp/tmp.olQmDTw8WB
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ mktemp
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.4cbjrmOW1u
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get ns
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.5srTFwL1JJ
+ cat /tmp/tmp.olQmDTw8WB
+ rm /tmp/tmp.5srTFwL1JJ /tmp/tmp.olQmDTw8WB
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.JTUEY2rNoM
+ cat /tmp/tmp.4cbjrmOW1u
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.JTUEY2rNoM /tmp/tmp.4cbjrmOW1u
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
++ mktemp
+ local LAST_OUT=/tmp/tmp.WAggzmrEeK
++ mktemp
+ local LAST_ERR=/tmp/tmp.CXQ2hBQv32
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.WAggzmrEeK
+ cat /tmp/tmp.CXQ2hBQv32
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found
+ rm /tmp/tmp.WAggzmrEeK /tmp/tmp.CXQ2hBQv32
+ return 1
+ true
+ kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth
Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found
Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found
+ true
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ kubectl_bin delete namespace clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.XawGtJ1WR5
++ mktemp
+ local LAST_ERR=/tmp/tmp.nNmhBIHFS5
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace clone-cluster-20561
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace clone-cluster-20561
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete namespace clone-cluster-20561
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.XawGtJ1WR5
+ cat /tmp/tmp.nNmhBIHFS5
Error from server (NotFound): namespaces "clone-cluster-20561" not found
+ rm /tmp/tmp.XawGtJ1WR5 /tmp/tmp.nNmhBIHFS5
+ return 1
+ :
+ wait_for_delete namespace/clone-cluster-20561
+ local res=namespace/clone-cluster-20561
+ set +o xtrace
namespace/clone-cluster-20561 - Error from server (NotFound): namespaces "clone-cluster-20561" not found
+ kubectl_bin create namespace clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.LL8WsTQqYT
++ mktemp
+ local LAST_ERR=/tmp/tmp.kJn7OgmcYV
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl create namespace clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.LL8WsTQqYT
namespace/clone-cluster-20561 created
+ cat /tmp/tmp.kJn7OgmcYV
+ rm /tmp/tmp.LL8WsTQqYT /tmp/tmp.kJn7OgmcYV
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dwgC964lH8
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9gjlVrttoT
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl config current-context
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.dwgC964lH8
++ cat /tmp/tmp.9gjlVrttoT
++ rm /tmp/tmp.dwgC964lH8 /tmp/tmp.9gjlVrttoT
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.c3Fpj9eeZY
++ mktemp
+ local LAST_ERR=/tmp/tmp.0ZxVdaL7ya
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox --namespace=clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.c3Fpj9eeZY
Context "gke_cloud-dev-112233_us-central1-c_jen-pg-721-a67b3475b-2-sandbox" modified.
+ cat /tmp/tmp.0ZxVdaL7ya
+ rm /tmp/tmp.c3Fpj9eeZY /tmp/tmp.0ZxVdaL7ya
+ return 0
+ deploy_operator
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local action=install
+ local namespace=clone-cluster-20561
+ local dont_send_telemetry=false
+ local version_service_uri_override=
+ desc 'start operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start operator
-----------------------------------------------------------------------------------

+ prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml disabled install clone-cluster-20561 false
+ local pull_secret_name=
+ local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml
+ local namespace_mode=disabled
+ local operator_action=install
+ local namespace=clone-cluster-20561
+ local dont_send_telemetry=false
+ yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml 'data[values.yaml]'
+ /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "clone-cluster-20561"#g'
+ /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g'
+ /usr/bin/sed -e 's#^namespace: .*#namespace: "clone-cluster-20561"#g'
+ /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g'
+ cat /dev/fd/63 -
++ echo 'crunchy_debug: "true"'
++ cut -d: -f1
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
++ echo perconalab/percona-postgresql-operator:PR-721-a67b3475b
++ cut -d: -f2
+ /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g'
+ /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-721-a67b3475b"#g'
+ [[ -n '' ]]
+ [[ GKE == \O\P\E\N\S\H\I\F\T ]]
+ yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/operator.yaml metadata.namespace clone-cluster-20561
+ yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-deployer
+ yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install
+ yq w -d3 - 'subjects[0].namespace' clone-cluster-20561
++ cat /tmp/tmp.UloFqeshVN/operator.ini
+ yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true"
archive_mode: "true"
archive_timeout: "60"
ccp_image_pull_secret: ""
ccp_image_pull_secret_manifest: ""
create_rbac: "true"
delete_operator_namespace: "false"
delete_watched_namespaces: "false"
disable_telemetry: "false"
namespace: "clone-cluster-20561"
namespace_mode: "disabled"
pgo_image_prefix: "perconalab/percona-postgresql-operator"
pgo_image_pull_policy: "Always"
pgo_image_pull_secret: ""
pgo_image_pull_secret_manifest: ""
pgo_image_tag: "PR-721-a67b3475b"
pgo_operator_namespace: "clone-cluster-20561"'
+ [[ -n '' ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ExWO1GjXzY
++ mktemp
+ local LAST_ERR=/tmp/tmp.De07pIkA50
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ExWO1GjXzY
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.De07pIkA50
+ rm /tmp/tmp.ExWO1GjXzY /tmp/tmp.De07pIkA50
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 23 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 24 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 25 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 26 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 27 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 28 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 29 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 30 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 31 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 32 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 33 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.UloFqeshVN/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.xWr4ZC9rYA
++ mktemp
+ local LAST_ERR=/tmp/tmp.DxCMP2RZfN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.UloFqeshVN/operator.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.xWr4ZC9rYA
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.DxCMP2RZfN
+ rm /tmp/tmp.xWr4ZC9rYA /tmp/tmp.DxCMP2RZfN
+ return 0
++ get_operator_pod
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.MAPsPFzH4d
+++ mktemp
++ local LAST_ERR=/tmp/tmp.MWSEWg3CK3
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.MAPsPFzH4d
++ cat /tmp/tmp.MWSEWg3CK3
++ rm /tmp/tmp.MAPsPFzH4d /tmp/tmp.MWSEWg3CK3
++ return 0
+ wait_pod postgres-operator-6b9f5c474d-88bgm
+ local pod=postgres-operator-6b9f5c474d-88bgm
+ local ns=
+ set +o xtrace
postgres-operator-6b9f5c474d-88bgmtrue
+ [[ -n '' ]]
+ desc 'create first PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PG cluster
-----------------------------------------------------------------------------------

+ cluster=some-name
+ spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local backup=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ local namespace=clone-cluster-20561
+ local root_via_pgbouncer=
+ desc 'create fresh PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create fresh PG cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name clone-cluster-20561
+ local cluster=some-name
+ local namespace=clone-cluster-20561
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - stringData.some-name some-name_pass
+ yq w - metadata.name some-name-users-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.oNg3xBTANu
++ mktemp
+ local LAST_ERR=/tmp/tmp.ohFwCUhjeN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.oNg3xBTANu
secret/some-name-users-secret created
+ cat /tmp/tmp.ohFwCUhjeN
+ rm /tmp/tmp.oNg3xBTANu /tmp/tmp.ohFwCUhjeN
+ return 0
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n clone-cluster-20561 apply -f -
+ yq w - metadata.name some-name-pmm-secret
++ mktemp
+ local LAST_OUT=/tmp/tmp.3RljLRQq1V
++ mktemp
+ local LAST_ERR=/tmp/tmp.cIDU5sLHDd
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.3RljLRQq1V
secret/some-name-pmm-secret created
+ cat /tmp/tmp.cIDU5sLHDd
+ rm /tmp/tmp.3RljLRQq1V /tmp/tmp.cIDU5sLHDd
+ return 0
+ kubectl_bin -n clone-cluster-20561 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.53ZZyedQ1k
++ mktemp
+ local LAST_ERR=/tmp/tmp.W421bGD1WC
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.53ZZyedQ1k
deployment.apps/pg-client created
+ cat /tmp/tmp.W421bGD1WC
+ rm /tmp/tmp.53ZZyedQ1k /tmp/tmp.W421bGD1WC
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name false false false false false false false false
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local root_via_pgbouncer=
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ yq w - metadata.labels.deployment-name some-name
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - spec.user some-name
+ yq w - spec.clustername some-name
+ yq w - metadata.name some-name
+ yq w - spec.secretsName some-name-users-secret
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - metadata.labels.name some-name
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.keepBackups false
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.keepData false
+ yq w - spec.name some-name
+ yq w - spec.database some-name
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.0ihJfTLyyj
++ mktemp
+ local LAST_ERR=/tmp/tmp.aJ1EmP4jVB
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.0ihJfTLyyj
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.aJ1EmP4jVB
+ rm /tmp/tmp.0ihJfTLyyj /tmp/tmp.aJ1EmP4jVB
+ return 0
+ wait_deployment some-name-backrest-shared-repo
+ local name=some-name-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-backrest-shared-repo.......
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name..........
+ [[ false == \g\c\s\+* ]]
+ wait_job_completion some-name-stanza-create false
+ local job=some-name-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
some-name-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name
+ local job=backrest-backup-some-name
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
backrest-backup-some-name+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=clone-cluster-20561
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lKIzZL7heQ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.586izauF05
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.lKIzZL7heQ
++ cat /tmp/tmp.586izauF05
++ rm /tmp/tmp.lKIzZL7heQ /tmp/tmp.586izauF05
++ return 0
+ status_query='kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.66o5537CS3
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.obIjIwS2bk
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.66o5537CS3
+++ cat /tmp/tmp.obIjIwS2bk
+++ rm /tmp/tmp.66o5537CS3 /tmp/tmp.obIjIwS2bk
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 1 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.gIgh2XZRCW
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.dAgngxvgMK
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.gIgh2XZRCW
+++ cat /tmp/tmp.dAgngxvgMK
+++ rm /tmp/tmp.gIgh2XZRCW /tmp/tmp.dAgngxvgMK
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6jElmFzMps
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.NsnlPWMT34
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.6jElmFzMps
+++ cat /tmp/tmp.NsnlPWMT34
+++ rm /tmp/tmp.6jElmFzMps /tmp/tmp.NsnlPWMT34
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.crgqdBjVqy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.N9PcLAHxUd
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.crgqdBjVqy
+++ cat /tmp/tmp.N9PcLAHxUd
+++ rm /tmp/tmp.crgqdBjVqy /tmp/tmp.N9PcLAHxUd
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2fxILkGhv5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.DjUDVpsrvz
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.2fxILkGhv5
+++ cat /tmp/tmp.DjUDVpsrvz
+++ rm /tmp/tmp.2fxILkGhv5 /tmp/tmp.DjUDVpsrvz
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.pKFa95PIb0
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.HbtNVX94NA
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.pKFa95PIb0
++++ cat /tmp/tmp.HbtNVX94NA
++++ rm /tmp/tmp.pKFa95PIb0 /tmp/tmp.HbtNVX94NA
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.HbBM5XfzlX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.TQAUQB96kK
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.HbBM5XfzlX
+++ cat /tmp/tmp.TQAUQB96kK
+++ rm /tmp/tmp.HbBM5XfzlX /tmp/tmp.TQAUQB96kK
+++ return 0
++ echo postgres_password
+ run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dVMr5ojFTt
+++ mktemp
++ local LAST_ERR=/tmp/tmp.4eQxgv1Mkx
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.dVMr5ojFTt
++ cat /tmp/tmp.4eQxgv1Mkx
++ rm /tmp/tmp.dVMr5ojFTt /tmp/tmp.4eQxgv1Mkx
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.MWJgIjwjI4
++ mktemp
+ local LAST_ERR=/tmp/tmp.gcYYt8UcsN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.MWJgIjwjI4
+ cat /tmp/tmp.gcYYt8UcsN
+ rm /tmp/tmp.MWJgIjwjI4 /tmp/tmp.gcYYt8UcsN
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZzblVULusF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.MPmW5aEbBM
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ZzblVULusF
+++ cat /tmp/tmp.MPmW5aEbBM
+++ rm /tmp/tmp.ZzblVULusF /tmp/tmp.MPmW5aEbBM
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.GCXA0HEd4I
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S8s7ySwINn
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.GCXA0HEd4I
+++ cat /tmp/tmp.S8s7ySwINn
+++ rm /tmp/tmp.GCXA0HEd4I /tmp/tmp.S8s7ySwINn
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.qPiO0PKnxc
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.6eiaG9Rebz
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.qPiO0PKnxc
+++ cat /tmp/tmp.6eiaG9Rebz
+++ rm /tmp/tmp.qPiO0PKnxc /tmp/tmp.6eiaG9Rebz
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.vmbHVMvsGc
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.qOBvDI6GdP
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.vmbHVMvsGc
++++ cat /tmp/tmp.qOBvDI6GdP
++++ rm /tmp/tmp.vmbHVMvsGc /tmp/tmp.qOBvDI6GdP
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.raZ7BqPe4k
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.7lY4Sq0RlX
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.raZ7BqPe4k
+++ cat /tmp/tmp.7lY4Sq0RlX
+++ rm /tmp/tmp.raZ7BqPe4k /tmp/tmp.7lY4Sq0RlX
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nn2i4WksC5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.UId1zQ5974
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.nn2i4WksC5
++ cat /tmp/tmp.UId1zQ5974
++ rm /tmp/tmp.nn2i4WksC5 /tmp/tmp.UId1zQ5974
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.cKseaDTmSn
++ mktemp
+ local LAST_ERR=/tmp/tmp.OX19mx5l4j
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.cKseaDTmSn
+ cat /tmp/tmp.OX19mx5l4j
+ rm /tmp/tmp.cKseaDTmSn /tmp/tmp.OX19mx5l4j
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SRNSErgOCV
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.uC5nZGijS1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.SRNSErgOCV
+++ cat /tmp/tmp.uC5nZGijS1
+++ rm /tmp/tmp.SRNSErgOCV /tmp/tmp.uC5nZGijS1
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JdeNnZ1J5c
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ePpyzsagof
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.JdeNnZ1J5c
+++ cat /tmp/tmp.ePpyzsagof
+++ rm /tmp/tmp.JdeNnZ1J5c /tmp/tmp.ePpyzsagof
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SoI1VcW5A7
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.wXrfbSMspp
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.SoI1VcW5A7
+++ cat /tmp/tmp.wXrfbSMspp
+++ rm /tmp/tmp.SoI1VcW5A7 /tmp/tmp.wXrfbSMspp
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.KMKgPoCWSd
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.bgJ7gSic3L
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.KMKgPoCWSd
++++ cat /tmp/tmp.bgJ7gSic3L
++++ rm /tmp/tmp.KMKgPoCWSd /tmp/tmp.bgJ7gSic3L
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eeC9OFW1kM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.bGyVrhX5nC
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.eeC9OFW1kM
+++ cat /tmp/tmp.bGyVrhX5nC
+++ rm /tmp/tmp.eeC9OFW1kM /tmp/tmp.bGyVrhX5nC
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.L8VkJnZcaL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.9bauwYIXgg
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.L8VkJnZcaL
++ cat /tmp/tmp.9bauwYIXgg
++ rm /tmp/tmp.L8VkJnZcaL /tmp/tmp.9bauwYIXgg
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Q2sD95s3k5
++ mktemp
+ local LAST_ERR=/tmp/tmp.RmqMkXJ9Vi
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Q2sD95s3k5
+ cat /tmp/tmp.RmqMkXJ9Vi
+ rm /tmp/tmp.Q2sD95s3k5 /tmp/tmp.RmqMkXJ9Vi
+ return 0
+ sleep 10
+ create_backup some-name fulldata full
+ local cluster=some-name
+ local bckp_prefix=fulldata
+ local bckp_type=full
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/backup.yml metadata.labels.pg-cluster some-name
+ yq w - metadata.name fulldata-some-name
+ yq w - spec.name fulldata-some-name
+ yq w - spec.namespace clone-cluster-20561
+ yq w - spec.parameters.pg-cluster some-name
+ kubectl_bin apply -f -
+ yq w --style=single -- - spec.parameters.backrest-opts --type=full
+ yq w - spec.parameters.job-name fulldata-some-name
++ mktemp
+ local LAST_OUT=/tmp/tmp.V2anGfoZ19
++ kubectl_bin get pods --selector=name=some-name-backrest-shared-repo,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
++ mktemp
+++ mktemp
+ local LAST_ERR=/tmp/tmp.Ojuurt7eTx
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ local LAST_OUT=/tmp/tmp.Aa8vEBvTDw
++ seq 0 2
+++ mktemp
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f -
++ local LAST_ERR=/tmp/tmp.vsIFgRanWL
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=some-name-backrest-shared-repo,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.Aa8vEBvTDw
++ cat /tmp/tmp.vsIFgRanWL
++ rm /tmp/tmp.Aa8vEBvTDw /tmp/tmp.vsIFgRanWL
++ return 0
+ yq w - spec.parameters.podname some-name-backrest-shared-repo-6f8f947dbc-c2lt7
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.V2anGfoZ19
pgtask.pg.percona.com/fulldata-some-name created
+ cat /tmp/tmp.Ojuurt7eTx
+ rm /tmp/tmp.V2anGfoZ19 /tmp/tmp.Ojuurt7eTx
+ return 0
+ sleep 10
+ wait_job_completion fulldata-some-name
+ local job=fulldata-some-name
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
fulldata-some-name+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ desc 'create clone from live cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create clone from live cluster
-----------------------------------------------------------------------------------

+ create_pgcluster_clone_from some-name
+ local cluster=some-name
+ local create_secrets=true
+ [[ true == \t\r\u\e ]]
+ create_user_secrets some-name-clone
+ local cluster=some-name-clone
+ local namespace=clone-cluster-20561
+ yq w - metadata.name some-name-clone-users-secret
+ yq w - stringData.some-name some-name_pass
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.eVMrztowDP
++ mktemp
+ local LAST_ERR=/tmp/tmp.GoAJ9rTNY9
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.eVMrztowDP
secret/some-name-clone-users-secret created
+ cat /tmp/tmp.GoAJ9rTNY9
+ rm /tmp/tmp.eVMrztowDP /tmp/tmp.GoAJ9rTNY9
+ return 0
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - metadata.name some-name-clone-pmm-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.E2SdGndpon
++ mktemp
+ local LAST_ERR=/tmp/tmp.KP7mYwETGs
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.E2SdGndpon
secret/some-name-clone-pmm-secret created
+ cat /tmp/tmp.KP7mYwETGs
+ rm /tmp/tmp.E2SdGndpon /tmp/tmp.KP7mYwETGs
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-clone local some-name
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name-clone
+ local backup=local
+ local restore_from=some-name
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=
+ local schedule=
+ local schedule_type=
+ local tls_only=
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-clone
+ yq w - metadata.labels.crunchy-pgha-scope some-name-clone
+ yq w - metadata.labels.deployment-name some-name-clone
+ yq w - spec.clustername some-name-clone
+ yq w - spec.database some-name-clone
+ yq w - metadata.name some-name-clone
+ yq w - metadata.labels.name some-name-clone
+ yq w - metadata.labels.pg-cluster some-name-clone
+ yq w - spec.user some-name
+ yq w - spec.keepBackups false
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.keepData false
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.name some-name-clone
+ yq w - spec.secretsName some-name-clone-users-secret
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ some-name != \f\a\l\s\e ]]
+ yq w /tmp/tmp.UloFqeshVN/cr.yaml spec.pgDataSource.restoreFrom some-name
+ mv /tmp/tmp.UloFqeshVN/cr.restore.yaml /tmp/tmp.UloFqeshVN/cr.yaml
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ '' == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.WEf1uv3Qv8
++ mktemp
+ local LAST_ERR=/tmp/tmp.kYkIQQP9TZ
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.WEf1uv3Qv8
perconapgcluster.pg.percona.com/some-name-clone created
+ cat /tmp/tmp.kYkIQQP9TZ
+ rm /tmp/tmp.WEf1uv3Qv8 /tmp/tmp.kYkIQQP9TZ
+ return 0
+ wait_bootstrap_completeness some-name-clone
+ local cluster=some-name-clone
+ wait_job_completion some-name-clone-bootstrap
+ local job=some-name-clone-bootstrap
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
some-name-clone-bootstrap+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_deployment some-name-clone-backrest-shared-repo
+ local name=some-name-clone-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone-backrest-shared-repo
+ wait_deployment some-name-clone
+ local name=some-name-clone
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone....
+ wait_job_completion some-name-clone-stanza-create false
+ local job=some-name-clone-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
some-name-clone-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name-clone
+ local job=backrest-backup-some-name-clone
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
backrest-backup-some-name-clone+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

++ get_psql_user_pass some-name some-name-clone
++ local user=some-name
++ local cluster=some-name-clone
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.VpO61w7qLe
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.B0Svn5DzK5
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.VpO61w7qLe
+++ cat /tmp/tmp.B0Svn5DzK5
+++ rm /tmp/tmp.VpO61w7qLe /tmp/tmp.B0Svn5DzK5
+++ return 0
++ [[ xsome-name-clone == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sJpxR7Dmk0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lxu7Q8SEPg
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.sJpxR7Dmk0
+++ cat /tmp/tmp.lxu7Q8SEPg
+++ rm /tmp/tmp.sJpxR7Dmk0 /tmp/tmp.lxu7Q8SEPg
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ZnAkZ0MvX5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1SH2lvQkcj
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ZnAkZ0MvX5
+++ cat /tmp/tmp.1SH2lvQkcj
+++ rm /tmp/tmp.ZnAkZ0MvX5 /tmp/tmp.1SH2lvQkcj
+++ return 0
++ [[ xsome-name-clone-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.LZ0jbONoqp
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.mplxBQm1we
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.LZ0jbONoqp
++++ cat /tmp/tmp.mplxBQm1we
++++ rm /tmp/tmp.LZ0jbONoqp /tmp/tmp.mplxBQm1we
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jCINj57s5Z
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ocVj5GG5aG
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.jCINj57s5Z
+++ cat /tmp/tmp.ocVj5GG5aG
+++ rm /tmp/tmp.jCINj57s5Z /tmp/tmp.ocVj5GG5aG
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.rqsIdQd4dw
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NiAKVa42qi
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.rqsIdQd4dw
++ cat /tmp/tmp.NiAKVa42qi
++ rm /tmp/tmp.rqsIdQd4dw /tmp/tmp.NiAKVa42qi
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.CzThKvWliL
++ mktemp
+ local LAST_ERR=/tmp/tmp.XGgzJUM4FA
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.CzThKvWliL
+ cat /tmp/tmp.XGgzJUM4FA
+ rm /tmp/tmp.CzThKvWliL /tmp/tmp.XGgzJUM4FA
+ return 0
+ '[' '!' -s /tmp/tmp.UloFqeshVN/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql /tmp/tmp.UloFqeshVN/select-1.sql
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepBackups","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.sEsWb4fUPV
++ mktemp
+ local LAST_ERR=/tmp/tmp.OSukFhR379
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepBackups","value":true}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.sEsWb4fUPV
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.OSukFhR379
+ rm /tmp/tmp.sEsWb4fUPV /tmp/tmp.OSukFhR379
+ return 0
+ sleep 30
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name
+ sleep 30
+ desc 'remove both clusters'
+ set +o xtrace


-----------------------------------------------------------------------------------
remove both clusters
-----------------------------------------------------------------------------------

+ kubectl delete perconapgclusters --all
perconapgcluster.pg.percona.com "some-name" deleted
perconapgcluster.pg.percona.com "some-name-clone" deleted
+ wait_for_delete deployment/some-name
+ local res=deployment/some-name
+ set +o xtrace
deployment/some-name - ......................................................................................................................Error from server (NotFound): deployments.apps "some-name" not found
+ wait_for_delete deployment/some-name-backrest-shared-repo
+ local res=deployment/some-name-backrest-shared-repo
+ set +o xtrace
deployment/some-name-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found
+ wait_for_delete deployment/some-name-clone
+ local res=deployment/some-name-clone
+ set +o xtrace
deployment/some-name-clone - Error from server (NotFound): deployments.apps "some-name-clone" not found
+ wait_for_delete deployment/some-name-clone-backrest-shared-repo
+ local res=deployment/some-name-clone-backrest-shared-repo
+ set +o xtrace
deployment/some-name-clone-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-clone-backrest-shared-repo" not found
+ desc 'start clone from backup only'
+ set +o xtrace


-----------------------------------------------------------------------------------
start clone from backup only
-----------------------------------------------------------------------------------

+ create_pgcluster_clone_from some-name
+ local cluster=some-name
+ local create_secrets=true
+ [[ true == \t\r\u\e ]]
+ create_user_secrets some-name-clone
+ local cluster=some-name-clone
+ local namespace=clone-cluster-20561
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - metadata.name some-name-clone-users-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
+ yq w - stringData.some-name some-name_pass
++ mktemp
+ local LAST_OUT=/tmp/tmp.DitGnmVn9L
++ mktemp
+ local LAST_ERR=/tmp/tmp.StbfQgcZBy
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.DitGnmVn9L
secret/some-name-clone-users-secret created
+ cat /tmp/tmp.StbfQgcZBy
+ rm /tmp/tmp.DitGnmVn9L /tmp/tmp.StbfQgcZBy
+ return 0
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - metadata.name some-name-clone-pmm-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.fjDc8HqFI9
++ mktemp
+ local LAST_ERR=/tmp/tmp.bCsB9XJeu9
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.fjDc8HqFI9
secret/some-name-clone-pmm-secret configured
+ cat /tmp/tmp.bCsB9XJeu9
+ rm /tmp/tmp.fjDc8HqFI9 /tmp/tmp.bCsB9XJeu9
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-clone local some-name
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name-clone
+ local backup=local
+ local restore_from=some-name
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=
+ local schedule=
+ local schedule_type=
+ local tls_only=
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-clone
+ yq w - metadata.labels.deployment-name some-name-clone
+ yq w - metadata.labels.name some-name-clone
+ yq w - spec.database some-name-clone
+ yq w - spec.keepData false
+ yq w - spec.keepBackups false
+ yq w - spec.name some-name-clone
+ yq w - metadata.labels.pg-cluster some-name-clone
+ yq w - spec.clustername some-name-clone
+ yq w - spec.secretsName some-name-clone-users-secret
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - metadata.name some-name-clone
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - metadata.labels.crunchy-pgha-scope some-name-clone
+ yq w - spec.user some-name
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ some-name != \f\a\l\s\e ]]
+ yq w /tmp/tmp.UloFqeshVN/cr.yaml spec.pgDataSource.restoreFrom some-name
+ mv /tmp/tmp.UloFqeshVN/cr.restore.yaml /tmp/tmp.UloFqeshVN/cr.yaml
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ '' == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.19K7dNOfEx
++ mktemp
+ local LAST_ERR=/tmp/tmp.45gU1ARNf5
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.19K7dNOfEx
perconapgcluster.pg.percona.com/some-name-clone created
+ cat /tmp/tmp.45gU1ARNf5
+ rm /tmp/tmp.19K7dNOfEx /tmp/tmp.45gU1ARNf5
+ return 0
+ wait_bootstrap_completeness some-name-clone
+ local cluster=some-name-clone
+ wait_job_completion some-name-clone-bootstrap
+ local job=some-name-clone-bootstrap
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 2 -ge 120 ]]
+ [[ 2 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 3 -ge 120 ]]
+ [[ 3 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 4 -ge 120 ]]
+ [[ 4 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 5 -ge 120 ]]
+ [[ 5 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 6 -ge 120 ]]
+ [[ 6 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 7 -ge 120 ]]
+ [[ 7 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 8 -ge 120 ]]
+ [[ 8 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
some-name-clone-bootstrap+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 22 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_deployment some-name-clone-backrest-shared-repo
+ local name=some-name-clone-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone-backrest-shared-repo
+ wait_deployment some-name-clone
+ local name=some-name-clone
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone
+ wait_job_completion some-name-clone-stanza-create false
+ local job=some-name-clone-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
some-name-clone-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name-clone
+ local job=backrest-backup-some-name-clone
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
backrest-backup-some-name-clone+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

++ get_psql_user_pass some-name some-name-clone
++ local user=some-name
++ local cluster=some-name-clone
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.9fzzu6egd4
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.QoHKAg3vhn
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.9fzzu6egd4
+++ cat /tmp/tmp.QoHKAg3vhn
+++ rm /tmp/tmp.9fzzu6egd4 /tmp/tmp.QoHKAg3vhn
+++ return 0
++ [[ xsome-name-clone == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fHHQUuSQOx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.o9AlsjAydv
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 8
+++ set -o errexit
+++ cat /tmp/tmp.fHHQUuSQOx
+++ cat /tmp/tmp.o9AlsjAydv
Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found
+++ rm /tmp/tmp.fHHQUuSQOx /tmp/tmp.o9AlsjAydv
+++ return 1
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.yLLk7ghTQn
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rbIBbUwQ7n
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.yLLk7ghTQn
+++ cat /tmp/tmp.rbIBbUwQ7n
+++ rm /tmp/tmp.yLLk7ghTQn /tmp/tmp.rbIBbUwQ7n
+++ return 0
++ [[ xsome-name-clone-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.cik7G2L5uv
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.iGKNHNjjHe
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.cik7G2L5uv
++++ cat /tmp/tmp.iGKNHNjjHe
++++ rm /tmp/tmp.cik7G2L5uv /tmp/tmp.iGKNHNjjHe
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.T5jqYZ6nZY
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.taOt8fHBnZ
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.T5jqYZ6nZY
+++ cat /tmp/tmp.taOt8fHBnZ
+++ rm /tmp/tmp.T5jqYZ6nZY /tmp/tmp.taOt8fHBnZ
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GsN8GPPgf0
+++ mktemp
++ local LAST_ERR=/tmp/tmp.lTrsooIoGR
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.GsN8GPPgf0
++ cat /tmp/tmp.lTrsooIoGR
++ rm /tmp/tmp.GsN8GPPgf0 /tmp/tmp.lTrsooIoGR
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.2ruIGdYH8B
++ mktemp
+ local LAST_ERR=/tmp/tmp.1mcuaawwqI
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2ruIGdYH8B
+ cat /tmp/tmp.1mcuaawwqI
+ rm /tmp/tmp.2ruIGdYH8B /tmp/tmp.1mcuaawwqI
+ return 0
+ '[' '!' -s /tmp/tmp.UloFqeshVN/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql /tmp/tmp.UloFqeshVN/select-1.sql
+ kubectl delete perconapgclusters --all
perconapgcluster.pg.percona.com "some-name-clone" deleted
+ wait_for_delete deployment/some-name
+ local res=deployment/some-name
+ set +o xtrace
deployment/some-name - Error from server (NotFound): deployments.apps "some-name" not found
+ wait_for_delete deployment/some-name-backrest-shared-repo
+ local res=deployment/some-name-backrest-shared-repo
+ set +o xtrace
deployment/some-name-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found
+ wait_for_delete deployment/some-name-clone
+ local res=deployment/some-name-clone
+ set +o xtrace
deployment/some-name-clone - Error from server (NotFound): deployments.apps "some-name-clone" not found
+ wait_for_delete deployment/some-name-clone-backrest-shared-repo
+ local res=deployment/some-name-clone-backrest-shared-repo
+ set +o xtrace
deployment/some-name-clone-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-clone-backrest-shared-repo" not found
+ kubectl_bin delete secret/some-name-users-secret
++ mktemp
+ local LAST_OUT=/tmp/tmp.WWvYEoRMjt
++ mktemp
+ local LAST_ERR=/tmp/tmp.eDTYfoZnGN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete secret/some-name-users-secret
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.WWvYEoRMjt
secret "some-name-users-secret" deleted
+ cat /tmp/tmp.eDTYfoZnGN
+ rm /tmp/tmp.WWvYEoRMjt /tmp/tmp.eDTYfoZnGN
+ return 0
+ kubectl delete pvc --all
persistentvolumeclaim "some-name-pgbr-repo" deleted
+ desc 'Create source cluster once again'
+ set +o xtrace


-----------------------------------------------------------------------------------
Create source cluster once again
-----------------------------------------------------------------------------------

+ spinup_pgcluster some-name /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local backup=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ local namespace=clone-cluster-20561
+ local root_via_pgbouncer=
+ desc 'create fresh PG cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create fresh PG cluster
-----------------------------------------------------------------------------------

+ create_user_secrets some-name clone-cluster-20561
+ local cluster=some-name
+ local namespace=clone-cluster-20561
+ yq w - metadata.name some-name-users-secret
+ yq w - stringData.some-name some-name_pass
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5DbGGZEthA
++ mktemp
+ local LAST_ERR=/tmp/tmp.eV3NyoN8MN
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.5DbGGZEthA
secret/some-name-users-secret created
+ cat /tmp/tmp.eV3NyoN8MN
+ rm /tmp/tmp.5DbGGZEthA /tmp/tmp.eV3NyoN8MN
+ return 0
+ yq w - metadata.name some-name-pmm-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.JvobKrlCrQ
++ mktemp
+ local LAST_ERR=/tmp/tmp.cUNeOfwzNX
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.JvobKrlCrQ
secret/some-name-pmm-secret configured
+ cat /tmp/tmp.cUNeOfwzNX
+ rm /tmp/tmp.JvobKrlCrQ /tmp/tmp.cUNeOfwzNX
+ return 0
+ kubectl_bin -n clone-cluster-20561 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.MJEVDT9tFo
++ mktemp
+ local LAST_ERR=/tmp/tmp.B2Ie7pdbq8
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/client.yml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.MJEVDT9tFo
deployment.apps/pg-client unchanged
+ cat /tmp/tmp.B2Ie7pdbq8
+ rm /tmp/tmp.MJEVDT9tFo /tmp/tmp.B2Ie7pdbq8
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name false false false false false false false false
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name
+ local backup=false
+ local restore_from=false
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=false
+ local schedule=false
+ local schedule_type=false
+ local tls_only=false
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name
+ yq w - metadata.labels.deployment-name some-name
+ yq w - metadata.labels.name some-name
+ yq w - spec.clustername some-name
+ yq w - spec.database some-name
+ yq w - metadata.name some-name
+ yq w - spec.name some-name
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.keepData false
+ yq w - spec.keepBackups false
+ yq w - spec.user some-name
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - metadata.labels.pg-cluster some-name
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.secretsName some-name-users-secret
+ yq w - metadata.labels.crunchy-pgha-scope some-name
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ false == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.uGbISS5GLg
++ mktemp
+ local LAST_ERR=/tmp/tmp.PQnh93eVsh
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.uGbISS5GLg
perconapgcluster.pg.percona.com/some-name created
+ cat /tmp/tmp.PQnh93eVsh
+ rm /tmp/tmp.uGbISS5GLg /tmp/tmp.PQnh93eVsh
+ return 0
+ wait_deployment some-name-backrest-shared-repo
+ local name=some-name-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-backrest-shared-repo...
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name..........
+ [[ false == \g\c\s\+* ]]
+ wait_job_completion some-name-stanza-create false
+ local job=some-name-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
some-name-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_job_completion backrest-backup-some-name
+ local job=backrest-backup-some-name
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
backrest-backup-some-name+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_cluster_consistency some-name
+ cluster_name=some-name
+ wait_cluster_status some-name 'pgcluster Initialized'
+ local cluster_name=some-name
+ local 'target_status=pgcluster Initialized'
+ local target_namespace=clone-cluster-20561
+ local status_query=
+ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.50hPCsGop6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Nv2PNMSFHP
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.50hPCsGop6
++ cat /tmp/tmp.Nv2PNMSFHP
++ rm /tmp/tmp.50hPCsGop6 /tmp/tmp.Nv2PNMSFHP
++ return 0
+ status_query='kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o jsonpath='\''{.status.PGCluster.state}'\'''
+ local retry=0
++ eval kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vDFdUOxudU
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.OBWvg0JuHu
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.vDFdUOxudU
+++ cat /tmp/tmp.OBWvg0JuHu
+++ rm /tmp/tmp.vDFdUOxudU /tmp/tmp.OBWvg0JuHu
+++ return 0
+ [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ let retry+=1
+ '[' 1 -ge 24 ']'
+ echo 'waiting for cluster readyness'
waiting for cluster readyness
+ sleep 10
++ eval kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath='\''{.status.PGCluster.state}'\'''
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.JhJJPrp3N2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.lMeW9WuGzr
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.status.PGCluster.state}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.JhJJPrp3N2
+++ cat /tmp/tmp.lMeW9WuGzr
+++ rm /tmp/tmp.JhJJPrp3N2 /tmp/tmp.lMeW9WuGzr
+++ return 0
+ [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]]
+ sleep 10
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.sUpCp8wITF
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.CtehPMQtEw
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.sUpCp8wITF
+++ cat /tmp/tmp.CtehPMQtEw
+++ rm /tmp/tmp.sUpCp8wITF /tmp/tmp.CtehPMQtEw
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.dgeiMEbx6G
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.5dRWE6TzyU
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.dgeiMEbx6G
+++ cat /tmp/tmp.5dRWE6TzyU
+++ rm /tmp/tmp.dgeiMEbx6G /tmp/tmp.5dRWE6TzyU
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.vpH1dCTk2r
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fr84pwfWRt
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.vpH1dCTk2r
+++ cat /tmp/tmp.fr84pwfWRt
+++ rm /tmp/tmp.vpH1dCTk2r /tmp/tmp.fr84pwfWRt
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.8Tyoqk58P0
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.WBDAZQDKAB
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.8Tyoqk58P0
++++ cat /tmp/tmp.WBDAZQDKAB
++++ rm /tmp/tmp.8Tyoqk58P0 /tmp/tmp.WBDAZQDKAB
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5QV2Lybjqt
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.weIXJdPP6t
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.5QV2Lybjqt
+++ cat /tmp/tmp.weIXJdPP6t
+++ rm /tmp/tmp.5QV2Lybjqt /tmp/tmp.weIXJdPP6t
+++ return 0
++ echo postgres_password
+ run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.hkb8T0hj6h
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vtoFEEO0SP
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.hkb8T0hj6h
++ cat /tmp/tmp.vtoFEEO0SP
++ rm /tmp/tmp.hkb8T0hj6h /tmp/tmp.vtoFEEO0SP
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.TvXOlmlMpv
++ mktemp
+ local LAST_ERR=/tmp/tmp.dYIn71pVZw
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.TvXOlmlMpv
+ cat /tmp/tmp.dYIn71pVZw
+ rm /tmp/tmp.TvXOlmlMpv /tmp/tmp.dYIn71pVZw
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Zr46pEPRgq
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.zgV9DP1Xaf
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.Zr46pEPRgq
+++ cat /tmp/tmp.zgV9DP1Xaf
+++ rm /tmp/tmp.Zr46pEPRgq /tmp/tmp.zgV9DP1Xaf
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.q6mEyBKMB2
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.YmMgYr0M7z
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.q6mEyBKMB2
+++ cat /tmp/tmp.YmMgYr0M7z
+++ rm /tmp/tmp.q6mEyBKMB2 /tmp/tmp.YmMgYr0M7z
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.LgTpZo1bCo
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9mkaUWDrei
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.LgTpZo1bCo
+++ cat /tmp/tmp.9mkaUWDrei
+++ rm /tmp/tmp.LgTpZo1bCo /tmp/tmp.9mkaUWDrei
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.CcK9PrNKwH
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.PZwrh8heXn
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.CcK9PrNKwH
++++ cat /tmp/tmp.PZwrh8heXn
++++ rm /tmp/tmp.CcK9PrNKwH /tmp/tmp.PZwrh8heXn
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.SbJ7LkafTL
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.u6zi0HC91T
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.SbJ7LkafTL
+++ cat /tmp/tmp.u6zi0HC91T
+++ rm /tmp/tmp.SbJ7LkafTL /tmp/tmp.u6zi0HC91T
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6WMMejRKei
+++ mktemp
++ local LAST_ERR=/tmp/tmp.0XdYS0sk5w
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.6WMMejRKei
++ cat /tmp/tmp.0XdYS0sk5w
++ rm /tmp/tmp.6WMMejRKei /tmp/tmp.0XdYS0sk5w
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.obgZ4nz9gL
++ mktemp
+ local LAST_ERR=/tmp/tmp.vjILWReRb0
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.obgZ4nz9gL
+ cat /tmp/tmp.vjILWReRb0
+ rm /tmp/tmp.obgZ4nz9gL /tmp/tmp.vjILWReRb0
+ return 0
++ get_psql_user_pass postgres some-name
++ local user=postgres
++ local cluster=some-name
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KiI54h29QI
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.VKIrQEz0UJ
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.KiI54h29QI
+++ cat /tmp/tmp.VKIrQEz0UJ
+++ rm /tmp/tmp.KiI54h29QI /tmp/tmp.VKIrQEz0UJ
+++ return 0
++ [[ xsome-name == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.X0mthBuzzh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.9yTxx58tEj
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.X0mthBuzzh
+++ cat /tmp/tmp.9yTxx58tEj
+++ rm /tmp/tmp.X0mthBuzzh /tmp/tmp.9yTxx58tEj
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.MW7dppdbWX
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Tx1FuHaWh1
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.MW7dppdbWX
+++ cat /tmp/tmp.Tx1FuHaWh1
+++ rm /tmp/tmp.MW7dppdbWX /tmp/tmp.Tx1FuHaWh1
+++ return 0
++ [[ xsome-name-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.MS1Nb7eV7K
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.Y9kEThccrs
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.MS1Nb7eV7K
++++ cat /tmp/tmp.Y9kEThccrs
++++ rm /tmp/tmp.MS1Nb7eV7K /tmp/tmp.Y9kEThccrs
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.zBDMQT9zDg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.cMMsoMO2KB
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-users-secret -o 'jsonpath={.data.postgres}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.zBDMQT9zDg
+++ cat /tmp/tmp.cMMsoMO2KB
+++ rm /tmp/tmp.zBDMQT9zDg /tmp/tmp.cMMsoMO2KB
+++ return 0
++ echo postgres_password
+ run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name.clone-cluster-20561
+ local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";'
+ local uri=postgres:postgres_password@some-name.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.oljhfKOGBI
+++ mktemp
++ local LAST_ERR=/tmp/tmp.49Vyh7BI5e
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.oljhfKOGBI
++ cat /tmp/tmp.49Vyh7BI5e
++ rm /tmp/tmp.oljhfKOGBI /tmp/tmp.49Vyh7BI5e
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.6arZwtkzII
++ mktemp
+ local LAST_ERR=/tmp/tmp.dFuOo5H4np
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.6arZwtkzII
+ cat /tmp/tmp.dFuOo5H4np
+ rm /tmp/tmp.6arZwtkzII /tmp/tmp.dFuOo5H4np
+ return 0
+ sleep 10
+ create_backup some-name fulldata full
+ local cluster=some-name
+ local bckp_prefix=fulldata
+ local bckp_type=full
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/conf/backup.yml metadata.labels.pg-cluster some-name
+ yq w - metadata.name fulldata-some-name
+ yq w --style=single -- - spec.parameters.backrest-opts --type=full
+ yq w - spec.parameters.job-name fulldata-some-name
+ kubectl_bin apply -f -
+ yq w - spec.name fulldata-some-name
+ yq w - spec.parameters.pg-cluster some-name
+ yq w - spec.namespace clone-cluster-20561
++ mktemp
++ kubectl_bin get pods --selector=name=some-name-backrest-shared-repo,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
+ local LAST_OUT=/tmp/tmp.gwUzptxf2c
++ mktemp
++ local LAST_OUT=/tmp/tmp.qI8L8RKfqt
+ local LAST_ERR=/tmp/tmp.n3EF6ih0z3
+ local exit_status=0
+ local timeout=4
+ set +o errexit
+++ mktemp
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f -
++ local LAST_ERR=/tmp/tmp.odMXYdS66K
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=some-name-backrest-shared-repo,pg-cluster=some-name -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.qI8L8RKfqt
++ cat /tmp/tmp.odMXYdS66K
++ rm /tmp/tmp.qI8L8RKfqt /tmp/tmp.odMXYdS66K
++ return 0
+ yq w - spec.parameters.podname some-name-backrest-shared-repo-6f8f947dbc-c79zl
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.gwUzptxf2c
pgtask.pg.percona.com/fulldata-some-name created
+ cat /tmp/tmp.n3EF6ih0z3
+ rm /tmp/tmp.gwUzptxf2c /tmp/tmp.n3EF6ih0z3
+ return 0
+ sleep 10
+ wait_job_completion fulldata-some-name
+ local job=fulldata-some-name
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
fulldata-some-name+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/fulldata-some-name -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ desc 'create clone from live cluster with secrets prepared by user'
+ set +o xtrace


-----------------------------------------------------------------------------------
create clone from live cluster with secrets prepared by user
-----------------------------------------------------------------------------------

+ create_pgcluster_clone_from some-name false
+ local cluster=some-name
+ local create_secrets=false
+ [[ false == \t\r\u\e ]]
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-clone local some-name
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name-clone
+ local backup=local
+ local restore_from=some-name
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=
+ local schedule=
+ local schedule_type=
+ local tls_only=
+ local root_via_pgbouncer=
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-clone
+ yq w - metadata.labels.crunchy-pgha-scope some-name-clone
+ yq w - metadata.labels.deployment-name some-name-clone
+ yq w - metadata.labels.name some-name-clone
+ yq w - spec.name some-name-clone
+ yq w - metadata.labels.pg-cluster some-name-clone
+ yq w - metadata.name some-name-clone
+ yq w - spec.clustername some-name-clone
+ yq w - spec.database some-name-clone
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.secretsName some-name-clone-users-secret
+ yq w - spec.user some-name
+ yq w - spec.keepBackups false
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.keepData false
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ some-name != \f\a\l\s\e ]]
+ yq w /tmp/tmp.UloFqeshVN/cr.yaml spec.pgDataSource.restoreFrom some-name
+ mv /tmp/tmp.UloFqeshVN/cr.restore.yaml /tmp/tmp.UloFqeshVN/cr.yaml
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ '' == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.8qMIAROfJZ
++ mktemp
+ local LAST_ERR=/tmp/tmp.YvTQvsJoJr
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.8qMIAROfJZ
perconapgcluster.pg.percona.com/some-name-clone created
+ cat /tmp/tmp.YvTQvsJoJr
+ rm /tmp/tmp.8qMIAROfJZ /tmp/tmp.YvTQvsJoJr
+ return 0
+ wait_bootstrap_completeness some-name-clone
+ local cluster=some-name-clone
+ wait_job_completion some-name-clone-bootstrap
+ local job=some-name-clone-bootstrap
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
some-name-clone-bootstrap+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_deployment some-name-clone-backrest-shared-repo
+ local name=some-name-clone-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone-backrest-shared-repo
+ wait_deployment some-name-clone
+ local name=some-name-clone
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone....
+ wait_job_completion some-name-clone-stanza-create false
+ local job=some-name-clone-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
some-name-clone-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_job_completion backrest-backup-some-name-clone
+ local job=backrest-backup-some-name-clone
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
backrest-backup-some-name-clone+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

++ get_psql_user_pass some-name some-name-clone
++ local user=some-name
++ local cluster=some-name-clone
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.ME8XxDX9gx
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.xZR8YzQvCc
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.ME8XxDX9gx
+++ cat /tmp/tmp.xZR8YzQvCc
+++ rm /tmp/tmp.ME8XxDX9gx /tmp/tmp.xZR8YzQvCc
+++ return 0
++ [[ xsome-name-clone == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.skmDV9H8nS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.rdz8zQOHbh
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.skmDV9H8nS
+++ cat /tmp/tmp.rdz8zQOHbh
+++ rm /tmp/tmp.skmDV9H8nS /tmp/tmp.rdz8zQOHbh
+++ return 0
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.jd5cT1zAx5
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.uKOKZt3xUE
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.jd5cT1zAx5
+++ cat /tmp/tmp.uKOKZt3xUE
+++ rm /tmp/tmp.jd5cT1zAx5 /tmp/tmp.uKOKZt3xUE
+++ return 0
++ [[ xsome-name-clone-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.XeNmaLeSIG
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.bzYAKvVddJ
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.XeNmaLeSIG
++++ cat /tmp/tmp.bzYAKvVddJ
++++ rm /tmp/tmp.XeNmaLeSIG /tmp/tmp.bzYAKvVddJ
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.azYzOBQTU0
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.ixaVFWDP6U
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.azYzOBQTU0
+++ cat /tmp/tmp.ixaVFWDP6U
+++ rm /tmp/tmp.azYzOBQTU0 /tmp/tmp.ixaVFWDP6U
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.7zVfGZ9kMU
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vTtKO1CeDT
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.7zVfGZ9kMU
++ cat /tmp/tmp.vTtKO1CeDT
++ rm /tmp/tmp.7zVfGZ9kMU /tmp/tmp.vTtKO1CeDT
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.ZKRbDepexN
++ mktemp
+ local LAST_ERR=/tmp/tmp.lxADHODJRB
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ZKRbDepexN
+ cat /tmp/tmp.lxADHODJRB
+ rm /tmp/tmp.ZKRbDepexN /tmp/tmp.lxADHODJRB
+ return 0
+ '[' '!' -s /tmp/tmp.UloFqeshVN/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql /tmp/tmp.UloFqeshVN/select-1.sql
+ kubectl_bin patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepBackups","value":true}]'
++ mktemp
+ local LAST_OUT=/tmp/tmp.Dvf2twGVnU
++ mktemp
+ local LAST_ERR=/tmp/tmp.3hpDZWFyAG
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl patch perconapgcluster/some-name --type json -p '[{"op":"add","path":"/spec/keepBackups","value":true}]'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.Dvf2twGVnU
perconapgcluster.pg.percona.com/some-name patched
+ cat /tmp/tmp.3hpDZWFyAG
+ rm /tmp/tmp.Dvf2twGVnU /tmp/tmp.3hpDZWFyAG
+ return 0
+ sleep 30
+ wait_deployment some-name
+ local name=some-name
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name
+ sleep 30
+ desc 'remove both clusters'
+ set +o xtrace


-----------------------------------------------------------------------------------
remove both clusters
-----------------------------------------------------------------------------------

+ kubectl delete perconapgclusters --all
perconapgcluster.pg.percona.com "some-name" deleted
perconapgcluster.pg.percona.com "some-name-clone" deleted
+ wait_for_delete deployment/some-name
+ local res=deployment/some-name
+ set +o xtrace
deployment/some-name - ..................Error from server (NotFound): deployments.apps "some-name" not found
+ wait_for_delete deployment/some-name-backrest-shared-repo
+ local res=deployment/some-name-backrest-shared-repo
+ set +o xtrace
deployment/some-name-backrest-shared-repo - ..Error from server (NotFound): deployments.apps "some-name-backrest-shared-repo" not found
+ wait_for_delete deployment/some-name-clone
+ local res=deployment/some-name-clone
+ set +o xtrace
deployment/some-name-clone - Error from server (NotFound): deployments.apps "some-name-clone" not found
+ wait_for_delete deployment/some-name-clone-backrest-shared-repo
+ local res=deployment/some-name-clone-backrest-shared-repo
+ set +o xtrace
deployment/some-name-clone-backrest-shared-repo - Error from server (NotFound): deployments.apps "some-name-clone-backrest-shared-repo" not found
+ desc 'start clone from backup only'
+ set +o xtrace


-----------------------------------------------------------------------------------
start clone from backup only
-----------------------------------------------------------------------------------

+ create_pgcluster_clone_from some-name
+ local cluster=some-name
+ local create_secrets=true
+ [[ true == \t\r\u\e ]]
+ create_user_secrets some-name-clone
+ local cluster=some-name-clone
+ local namespace=clone-cluster-20561
+ yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ yq w - metadata.name some-name-clone-users-secret
+ kubectl_bin -n clone-cluster-20561 apply -f -
+ yq w - stringData.some-name some-name_pass
++ mktemp
+ local LAST_OUT=/tmp/tmp.bddzSZvV2F
++ mktemp
+ local LAST_ERR=/tmp/tmp.qHfI26GjqX
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.bddzSZvV2F
secret/some-name-clone-users-secret created
+ cat /tmp/tmp.qHfI26GjqX
+ rm /tmp/tmp.bddzSZvV2F /tmp/tmp.qHfI26GjqX
+ return 0
+ yq w - metadata.name some-name-clone-pmm-secret
+ yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/users-secret.yaml
+ kubectl_bin -n clone-cluster-20561 apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.7DC2IrGjvX
++ mktemp
+ local LAST_ERR=/tmp/tmp.ds7L4IHQTC
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 apply -f -
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.7DC2IrGjvX
secret/some-name-clone-pmm-secret configured
+ cat /tmp/tmp.ds7L4IHQTC
+ rm /tmp/tmp.7DC2IrGjvX /tmp/tmp.ds7L4IHQTC
+ return 0
+ apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml some-name-clone local some-name
+ local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml
+ local name=some-name-clone
+ local backup=local
+ local restore_from=some-name
+ local restore_opts=false
+ local custom_config=false
+ local podAntiAffinity=
+ local schedule=
+ local schedule_type=
+ local tls_only=
+ local root_via_pgbouncer=
+ yq w - metadata.labels.crunchy-pgha-scope some-name-clone
+ yq w - metadata.labels.deployment-name some-name-clone
+ yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-721/deploy/cr.yaml metadata.annotations.current-primary some-name-clone
+ yq w - metadata.labels.pg-cluster some-name-clone
+ yq w - metadata.labels.name some-name-clone
+ yq w - spec.keepData false
+ yq w - spec.user some-name
+ yq w - spec.keepBackups false
+ yq w - metadata.name some-name-clone
+ yq w - spec.clustername some-name-clone
+ yq w - spec.database some-name-clone
+ yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo
+ yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger
+ yq w - spec.pmm.image perconalab/pmm-client:2.41.0
+ yq w - spec.name some-name-clone
+ yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer
+ yq w - spec.secretsName some-name-clone-users-secret
+ yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha
+ yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest
+ case ${podAntiAffinity} in
+ case ${backup} in
+ [[ some-name != \f\a\l\s\e ]]
+ yq w /tmp/tmp.UloFqeshVN/cr.yaml spec.pgDataSource.restoreFrom some-name
+ mv /tmp/tmp.UloFqeshVN/cr.restore.yaml /tmp/tmp.UloFqeshVN/cr.yaml
+ [[ false != \f\a\l\s\e ]]
+ [[ false != \f\a\l\s\e ]]
+ [[ '' == \t\r\u\e ]]
+ [[ '' == \t\r\u\e ]]
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
++ mktemp
+ local LAST_OUT=/tmp/tmp.7hDhBI6ini
++ mktemp
+ local LAST_ERR=/tmp/tmp.V6LmfT2Znh
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/cr.yaml -n clone-cluster-20561
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.7hDhBI6ini
perconapgcluster.pg.percona.com/some-name-clone created
+ cat /tmp/tmp.V6LmfT2Znh
+ rm /tmp/tmp.7hDhBI6ini /tmp/tmp.V6LmfT2Znh
+ return 0
+ wait_bootstrap_completeness some-name-clone
+ local cluster=some-name-clone
+ wait_job_completion some-name-clone-bootstrap
+ local job=some-name-clone-bootstrap
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 2 -ge 120 ]]
+ [[ 2 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 3 -ge 120 ]]
+ [[ 3 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 4 -ge 120 ]]
+ [[ 4 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 5 -ge 120 ]]
+ [[ 5 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 6 -ge 120 ]]
+ [[ 6 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 7 -ge 120 ]]
+ [[ 7 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 8 -ge 120 ]]
+ [[ 8 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 9 -ge 120 ]]
+ [[ 9 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
some-name-clone-bootstrap+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 15 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 16 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 17 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 18 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 19 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 20 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 21 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-bootstrap -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ not found == \n\o\t\ \f\o\u\n\d ]]
+ echo

+ wait_deployment some-name-clone-backrest-shared-repo
+ local name=some-name-clone-backrest-shared-repo
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone-backrest-shared-repo
+ wait_deployment some-name-clone
+ local name=some-name-clone
+ local target_namespace=clone-cluster-20561
+ sleep 10
+ set +o xtrace
some-name-clone....
+ wait_job_completion some-name-clone-stanza-create false
+ local job=some-name-clone-stanza-create
+ local mandatory=false
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
some-name-clone-stanza-create+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/some-name-clone-stanza-create -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ wait_job_completion backrest-backup-some-name-clone
+ local job=backrest-backup-some-name-clone
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
+ sleep 1
+ echo -n .
.+ let retry+=1
+ [[ 1 -ge 120 ]]
+ [[ 1 -ge 120 ]]
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
backrest-backup-some-name-clone+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/backrest-backup-some-name-clone -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

++ get_psql_user_pass some-name some-name-clone
++ local user=some-name
++ local cluster=some-name-clone
++ local namespace=clone-cluster-20561
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.k4qByrNuBl
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.q1K0P9hXeb
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.metadata.name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.k4qByrNuBl
+++ cat /tmp/tmp.q1K0P9hXeb
+++ rm /tmp/tmp.k4qByrNuBl /tmp/tmp.q1K0P9hXeb
+++ return 0
++ [[ xsome-name-clone == \x ]]
++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in
+++ kubectl_bin get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.wzTggjorpv
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.3LeT5qGUBE
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 0
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 4
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl get perconapgcluster/some-name -o 'jsonpath={.metadata.labels.pgo-version}'
+++ exit_status=1
+++ [[ 1 != 0 ]]
+++ sleep 8
+++ set -o errexit
+++ cat /tmp/tmp.wzTggjorpv
+++ cat /tmp/tmp.3LeT5qGUBE
Error from server (NotFound): perconapgclusters.pg.percona.com "some-name" not found
+++ rm /tmp/tmp.wzTggjorpv /tmp/tmp.3LeT5qGUBE
+++ return 1
+++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.6E6Gn18rFZ
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.S9vpJxTCkW
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.6E6Gn18rFZ
+++ cat /tmp/tmp.S9vpJxTCkW
+++ rm /tmp/tmp.6E6Gn18rFZ /tmp/tmp.S9vpJxTCkW
+++ return 0
++ [[ xsome-name-clone-users-secret == \x ]]
+++ base64 -d
++++ kubectl_bin -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
+++++ mktemp
++++ local LAST_OUT=/tmp/tmp.9IZAAcdWXQ
+++++ mktemp
++++ local LAST_ERR=/tmp/tmp.UpSPIwaN3E
++++ local exit_status=0
++++ local timeout=4
++++ set +o errexit
+++++ seq 0 2
++++ for i in '$(seq 0 2)'
++++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++++ kubectl -n clone-cluster-20561 get perconapgcluster/some-name-clone -o 'jsonpath={.spec.secretsName}'
++++ exit_status=0
++++ [[ 0 != 0 ]]
++++ break
++++ set -o errexit
++++ cat /tmp/tmp.9IZAAcdWXQ
++++ cat /tmp/tmp.UpSPIwaN3E
++++ rm /tmp/tmp.9IZAAcdWXQ /tmp/tmp.UpSPIwaN3E
++++ return 0
+++ kubectl_bin -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.Fc1JD56rlS
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.72ZUiFqU8b
+++ local exit_status=0
+++ local timeout=4
+++ set +o errexit
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+++ kubectl -n clone-cluster-20561 get secret/some-name-clone-users-secret -o 'jsonpath={.data.some-name}'
+++ exit_status=0
+++ [[ 0 != 0 ]]
+++ break
+++ set -o errexit
+++ cat /tmp/tmp.Fc1JD56rlS
+++ cat /tmp/tmp.72ZUiFqU8b
+++ rm /tmp/tmp.Fc1JD56rlS /tmp/tmp.72ZUiFqU8b
+++ return 0
++ echo some-name_pass
+ compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local command_id=select-1
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local postfix=
+ local uri_suffix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql
+ run_psql '\c myapp \\\ SELECT * from myApp;' some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561 postgres ''
+ local 'command=\c myapp \\\ SELECT * from myApp;'
+ local uri=some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561
+ local driver=postgres
+ local suffix=.svc.cluster.local
++ kubectl_bin -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.XuqIROahKn
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zlS1mZ2tkw
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl -n clone-cluster-20561 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.XuqIROahKn
++ cat /tmp/tmp.zlS1mZ2tkw
++ rm /tmp/tmp.XuqIROahKn /tmp/tmp.zlS1mZ2tkw
++ return 0
+ local client_container=pg-client-76df9df876-dl96n
+ kubectl_bin exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
++ mktemp
+ local LAST_OUT=/tmp/tmp.4zW0bM5JHc
++ mktemp
+ local LAST_ERR=/tmp/tmp.4Kpbnp338C
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl exec pg-client-76df9df876-dl96n -n clone-cluster-20561 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://some-name:some-name_pass@some-name-clone-pgbouncer.clone-cluster-20561.svc.cluster.local'
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.4zW0bM5JHc
+ cat /tmp/tmp.4Kpbnp338C
+ rm /tmp/tmp.4zW0bM5JHc /tmp/tmp.4Kpbnp338C
+ return 0
+ '[' '!' -s /tmp/tmp.UloFqeshVN/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-721/e2e-tests/clone-cluster/compare/select-1.sql /tmp/tmp.UloFqeshVN/select-1.sql
+ destroy clone-cluster-20561
++ get_operator_pod
+ grep -v level=debug
++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
+ grep -v 'Getting tasks for pod'
+ grep -v 'Getting pods from source'
+ grep -v 'get backup status: Job.batch'
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+++ mktemp
+ grep -v 'the object has been modified'
+ grep -v level=info
+ sort -u
+ tee /tmp/tmp.UloFqeshVN/operator.log
++ local LAST_OUT=/tmp/tmp.m1rShq0aIR
+++ mktemp
++ local LAST_ERR=/tmp/tmp.kDlL4xxto4
++ local exit_status=0
++ local timeout=4
++ set +o errexit
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ [[ 0 != 0 ]]
++ break
++ set -o errexit
++ cat /tmp/tmp.m1rShq0aIR
++ cat /tmp/tmp.kDlL4xxto4
++ rm /tmp/tmp.m1rShq0aIR /tmp/tmp.kDlL4xxto4
++ return 0
+ kubectl_bin logs postgres-operator-6b9f5c474d-88bgm -c operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.B5lW5oFNwq
++ mktemp
+ local LAST_ERR=/tmp/tmp.jkNcEebUys
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl logs postgres-operator-6b9f5c474d-88bgm -c operator
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.B5lW5oFNwq
+ cat /tmp/tmp.jkNcEebUys
+ rm /tmp/tmp.B5lW5oFNwq /tmp/tmp.jkNcEebUys
+ return 0

  
    
        
            
                
                  
                    
                      
                        
                          
                            
                                
                                        
                                            
                                            ]
                                        {
                                        }
                                        },
                                    ]
                                    }
                                    },
                                  {
                                  }
                                  },
                                ]
                                {
                                }
                                },
                              {
                              }
                            ]
                            {
                            }
                          {
                          }
                          },
                         ]
                         }
                        ]
                        {
                        }
                        },
                        }, {
                      ]
                      {
                      }
                      },
                     ]
                    ]
                    ],
                    {
                    }
                    },
                    },  {
                    }, {
                    },{
                    }]
                    }],
                  ]
                  {
                  }
                  },
                ,
                ],
                {
                }
                },
                }, {
                }],
            {
            }
            },
            }],
          }
        ],
        }
        },
    }
    },
  }
 },
{
}
},
	  },
		
2024/04/23 17:10:44 INF    1 (localhost:4150) connecting to nsqd
2024/04/23 17:11:45 INF    2 (localhost:4150) connecting to nsqd
2024/04/23 17:11:46 INF    3 (localhost:4150) connecting to nsqd
2024/04/23 17:11:52 INF    4 (localhost:4150) connecting to nsqd
2024/04/23 17:12:20 INF    5 (localhost:4150) connecting to nsqd
2024/04/23 17:12:20 INF    6 (localhost:4150) connecting to nsqd
2024/04/23 17:12:21 INF    7 (localhost:4150) connecting to nsqd
2024/04/23 17:13:07 INF    8 (localhost:4150) connecting to nsqd
2024/04/23 17:13:28 INF    9 (localhost:4150) connecting to nsqd
2024/04/23 17:14:12 INF   10 (localhost:4150) connecting to nsqd
2024/04/23 17:14:56 INF   11 (localhost:4150) connecting to nsqd
2024/04/23 17:14:57 INF   12 (localhost:4150) connecting to nsqd
2024/04/23 17:15:20 INF   13 (localhost:4150) connecting to nsqd
2024/04/23 17:15:39 INF   14 (localhost:4150) connecting to nsqd
2024/04/23 17:15:39 INF   15 (localhost:4150) connecting to nsqd
2024/04/23 17:15:39 INF   16 (localhost:4150) connecting to nsqd
2024/04/23 17:17:46 INF   17 (localhost:4150) connecting to nsqd
2024/04/23 17:17:47 INF   18 (localhost:4150) connecting to nsqd
2024/04/23 17:18:13 INF   19 (localhost:4150) connecting to nsqd
2024/04/23 17:18:18 INF   20 (localhost:4150) connecting to nsqd
2024/04/23 17:20:01 INF   21 (localhost:4150) connecting to nsqd
2024/04/23 17:20:06 INF   22 (localhost:4150) connecting to nsqd
2024/04/23 17:21:45 INF   23 (localhost:4150) connecting to nsqd
2024/04/23 17:22:30 INF   24 (localhost:4150) connecting to nsqd
2024/04/23 17:22:31 INF   25 (localhost:4150) connecting to nsqd
2024/04/23 17:22:37 INF   26 (localhost:4150) connecting to nsqd
2024/04/23 17:22:55 INF   27 (localhost:4150) connecting to nsqd
2024/04/23 17:22:55 INF   28 (localhost:4150) connecting to nsqd
2024/04/23 17:22:55 INF   29 (localhost:4150) connecting to nsqd
2024/04/23 17:23:13 INF   30 (localhost:4150) connecting to nsqd
2024/04/23 17:23:33 INF   31 (localhost:4150) connecting to nsqd
2024/04/23 17:23:38 INF   32 (localhost:4150) connecting to nsqd
2024/04/23 17:24:13 INF   33 (localhost:4150) connecting to nsqd
2024/04/23 17:25:05 INF   34 (localhost:4150) connecting to nsqd
2024/04/23 17:25:06 INF   35 (localhost:4150) connecting to nsqd
2024/04/23 17:25:12 INF   36 (localhost:4150) connecting to nsqd
2024/04/23 17:25:40 INF   37 (localhost:4150) connecting to nsqd
2024/04/23 17:25:40 INF   38 (localhost:4150) connecting to nsqd
2024/04/23 17:25:41 INF   39 (localhost:4150) connecting to nsqd
2024/04/23 17:26:27 INF   40 (localhost:4150) connecting to nsqd
2024/04/23 17:26:48 INF   41 (localhost:4150) connecting to nsqd
2024/04/23 17:27:27 INF   42 (localhost:4150) connecting to nsqd
2024/04/23 17:28:12 INF   43 (localhost:4150) connecting to nsqd
2024/04/23 17:28:12 INF   44 (localhost:4150) connecting to nsqd
2024/04/23 17:28:20 INF   45 (localhost:4150) connecting to nsqd
2024/04/23 17:28:38 INF   46 (localhost:4150) connecting to nsqd
2024/04/23 17:28:38 INF   47 (localhost:4150) connecting to nsqd
2024/04/23 17:28:39 INF   48 (localhost:4150) connecting to nsqd
2024/04/23 17:30:27 INF   49 (localhost:4150) connecting to nsqd
2024/04/23 17:30:28 INF   50 (localhost:4150) connecting to nsqd
2024/04/23 17:30:54 INF   51 (localhost:4150) connecting to nsqd
2024/04/23 17:30:59 INF   52 (localhost:4150) connecting to nsqd
2024/04/23 17:31:04 INF   53 (localhost:4150) connecting to nsqd
2024/04/23 17:31:10 INF   54 (localhost:4150) connecting to nsqd
2024/04/23 17:32:30 INF   55 (localhost:4150) connecting to nsqd
2024/04/23 17:33:14 INF   56 (localhost:4150) connecting to nsqd
2024/04/23 17:33:14 INF   57 (localhost:4150) connecting to nsqd
2024/04/23 17:33:21 INF   58 (localhost:4150) connecting to nsqd
2024/04/23 17:33:38 INF   59 (localhost:4150) connecting to nsqd
2024/04/23 17:33:39 INF   60 (localhost:4150) connecting to nsqd
2024/04/23 17:33:39 INF   61 (localhost:4150) connecting to nsqd
        "accessModes": [
                "affinity": 
                "affinity": {
                "affinity": {                 "podAntiAffinity": {
                      "allowPrivilegeEscalation": false,
                "annotations": {"keep-backups":"false","keep-data":"false"},
    "apiVersion": "apps/v1",
    "apiVersion": "batch/v1",
                                        "apiVersion": "v1",
        "apiVersion": "v1",
    "apiVersion": "v1",
                    "backrest-command": "backup",
                    "backrest-command": "stanza-create",
                        "claimName": "some-name-clone-pgbr-repo"
                        "claimName": "some-name-pgbr-repo"
                            "command": [
                                    "configMap": {
                                "configMap": {
                          "configMap": {
                            "configMapKeyRef": {
                                        "containerName": "database",
                        "containerPort": 2022,
                        "containerPort": 5432,
                        "containerPort": 8009,
                "containers": [
                "containers": [{
    "cpu": "1",
    "cpu": "200m",
    "cpu": "500m",
                                                "crunchydata"
                    "crunchy-pgbouncer": "true",
                "crunchy-pgbouncer": "true",
            "crunchy-pgbouncer": "true",
            "crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin"
                            "defaultMode": 288
                        "defaultMode": 288,
                    "defaultMode": 288,
                            "defaultMode": 420,
            "deployment-name": "some-name"
            "deployment-name": "some-name-clone"
            "deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone"
            "deployment-name": "some-name-clone-repl1"
            "deployment-name": "some-name-clone-repl2"
            "deployment-name": "some-name-repl1"
            "deployment-name": "some-name-repl2"
                    "deployment-name": "some-name-repl2","pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name"
                                        "divisor": "1m",
                "dnsPolicy": "ClusterFirst"
                        "downwardAPI": {
                        "emptyDir": {
                      "emptyDir": {
                    "emptyDir": {
                  "emptyDir": {
                    "env": [
                    "env": [{
                        "exec": {
                                        "fieldPath": "metadata.annotations"
                                        "fieldPath": "metadata.labels"
                                "fieldPath": "metadata.labels['crunchy-pgha-scope']"
                                "fieldPath": "metadata.namespace"
                                    "fieldRef": {
                            "fieldRef": {
                  "fsGroup": 2,
I0423 17:10:08.723789       1 shared_informer.go:240] Waiting for caches to sync for namespace
I0423 17:10:08.824317       1 shared_informer.go:247] Caches are synced for namespace 
I0423 17:10:08.919547       1 shared_informer.go:240] Waiting for caches to sync for clone-cluster-20561
I0423 17:10:09.019670       1 shared_informer.go:247] Caches are synced for clone-cluster-20561 
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer",
                    "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha",
                    "image": "perconalab/percona-postgresql-operator:PR-721-a67b3475b-pgo-rmdata",
                    "imagePullPolicy": "IfNotPresent"
                    "imagePullPolicy": "IfNotPresent",
                        "initialDelaySeconds": 15
                        "initialDelaySeconds": 30,
                                "items": [
                            "items": [
                        "items": [
                      "items": [
                                 "key": "config",
                             "key": "config",
                           "key": "config",
                                            "key": "crunchy-pgbouncer",
                               "key": "init"
                                "key": "password"
                                            "key": "pg-cluster",
                                            "key": "pgo-backrest",
                                            "key": "pgo-backrest-job",
                                            "key": "pgo-pg-database",
                                            "key": "pg-pod-anti-affinity",
                                    "key": "tls.crt",
                                    "key": "tls.key",
                                            "key": "vendor",
    "kind": "Deployment",
    "kind": "Job",
    "kind": "PersistentVolumeClaim",
        "kind": "Service",
                "labels": {
            "labels": {
        "labels": {
                                "labelSelector": {
                    "livenessProbe": {
                                    "matchExpressions": [
            "matchLabels": {
          "matchLabels": {
                "maxSurge": 1
                "maxUnavailable": 1,
                          "medium": "Memory",
                        "medium": "Memory"
                        "medium": "Memory",
                      "medium": "Memory",
                    "medium": "Memory",
    "memory": "128Mi"
    "memory": "256Mi"
    "memory": "48Mi"
            "metadata": {
        "metadata": {
    "metadata": {
                      "mountPath": "/backrestrepo",
                            "mountPath": "/dev/shm",
                        "mountPath": "/dev/shm",
                      "mountPath": "/etc/pgbackrest"
                            "mountPath": "/etc/pgbackrest/conf.d",
                        "mountPath": "/etc/pgbackrest/conf.d",
                            "mountPath": "/etc/podinfo",
                          "mountPath": "/etc/ssh",
                      "mountPath": "/etc/ssh",
                      "mountPath": "/home/pgbackrest"
                            "mountPath": "/pgconf",
                        "mountPath": "/pgconf/",
                        "mountPath": "/pgconf",
                            "mountPath": "/pgconf/pgreplicator",
                        "mountPath": "/pgconf/pgreplicator",
                            "mountPath": "/pgconf/pgsuper",
                        "mountPath": "/pgconf/pgsuper",
                            "mountPath": "/pgconf/pguser",
                        "mountPath": "/pgconf/pguser",
                          "mountPath": "/pgconf/tls",
                      "mountPath": "/pgconf/tls",
                        "mountPath": "/pgconf/tls/pgbouncer",
                            "mountPath": "/pgconf/tls-replication",
                            "mountPath": "/pgdata",
                        "mountPath": "/pgdata",
                            "mountPath": "/sshd",
                        "mountPath": "/sshd",
                      "mountPath": "/sshd",
                            "mountPath": "/tmp",
                          "mountPath": "/tmp",
                        "mountPath": "/tmp",
                      "mountPath": "/tmp"
                    "name": "backrest",
                "name": "backrest-backup-some-name",
                "name": "backrest-backup-some-name-clone",
                      "name": "backrestrepo",
                    "name": "backrestrepo",
  "name": "BACKREST_SKIP_CREATE_STANZA",
                                "name": "cluster1-replication-ssl-keypair",
                                "name": "cluster1-ssl-ca"
                              "name": "cluster1-ssl-ca"
                            "name": "cluster1-ssl-ca"
                                "name": "cluster1-ssl-keypair"
                              "name": "cluster1-ssl-keypair"
                            "name": "cluster1-ssl-keypair"
                        "name": "COMMAND",
                        "name": "COMMAND_OPTS",
                    "name": "database",
                            "name": "dshm"
                        "name": "dshm"
                      "name": "dshm",
                    "name": "dshm",
  "name": "ENABLE_SSHD",
                "name": "fulldata-some-name",
                        "name": "IS_BACKUP",
                        "name": "IS_REPLICA",
                        "name": "LD_PRELOAD",
                      "name": "LD_PRELOAD",
                        "name": "MODE",
                        "name": "NAMESPACE",
                        "name": "NSS_WRAPPER_GROUP",
                      "name": "NSS_WRAPPER_GROUP",
                        "name": "NSS_WRAPPER_PASSWD",
                      "name": "NSS_WRAPPER_PASSWD",
                        "name": "patroni"
                        "name": "PATRONI_KUBERNETES_LABELS",
                        "name": "PATRONI_KUBERNETES_NAMESPACE",
                        "name": "PATRONI_KUBERNETES_SCOPE_LABEL",
                        "name": "PATRONI_LOG_LEVEL",
                        "name": "PATRONI_POSTGRESQL_DATA_DIR",
                        "name": "PATRONI_SCOPE",
                        "name": "pgbackrest"
                      "name": "pgbackrest-conf",
                  "name": "pgbackrest-conf",
                            "name": "pgbackrest-config"
                        "name": "pgbackrest-config"
                      "name": "pgbackrest-config",
                    "name": "pgbackrest-config",
                        "name": "PGBACKREST_DB_HOST",
                        "name": "PGBACKREST_DB_PATH",
  "name": "PGBACKREST_DB_PATH",
                      "name": "pgbackrest-home",
                  "name": "pgbackrest-home",
                        "name": "PGBACKREST_LOG_PATH",
  "name": "PGBACKREST_LOG_PATH",
                        "name": "PGBACKREST_PG1_PORT",
  "name": "PGBACKREST_PG1_PORT",
                        "name": "PGBACKREST_PG1_SOCKET_PATH",
  "name": "PGBACKREST_PG1_SOCKET_PATH",
  "name": "PGBACKREST_REPO1_HOST",
                        "name": "PGBACKREST_REPO1_PATH",
  "name": "PGBACKREST_REPO1_PATH",
                        "name": "PGBACKREST_REPO1_TYPE",
  "name": "PGBACKREST_REPO1_TYPE",
                        "name": "PGBACKREST_STANZA",
  "name": "PGBACKREST_STANZA",
                        "name": "pgbouncer"
                    "name": "pgbouncer",
                        "name": "pgbouncer-conf",
                    "name": "pgbouncer-conf",
                        "name": "PG_CLUSTER",
                            "name": "pgconf-volume"
                        "name": "pgconf-volume"
                        "name": "pgconf-volume",
                    "name": "pgconf-volume",
                            "name": "pgdata",
                        "name": "pgdata"
                        "name": "pgdata",
                    "name": "pgdata",
                        "name": "PGHA_BOOTSTRAP_METHOD",
                        "name": "PGHA_DATABASE",
                        "name": "PGHA_INIT",
                        "name": "PGHA_PASSWORD_TYPE",
  "name": "PGHA_PGBACKREST",
                        "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
  "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE",
                        "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
  "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE",
                        "name": "PGHA_PGBACKREST_S3_VERIFY_TLS",
                        "name": "PGHA_PG_PORT",
                        "name": "PGHA_REPLICA_REINIT_ON_START_FAIL",
                        "name": "PGHA_SCOPE",
                        "name": "PGHA_STANDBY",
                        "name": "PGHA_SYNC_REPLICATION",
                        "name": "PGHA_TLS_ENABLED",
                        "name": "PGHA_TLS_ONLY",
                        "name": "PGHA_USER",
                        "name": "PGHOST",
                        "name": "PG_PASSWORD",
                        "name": "PG_PRIMARY_SERVICE_NAME",
                        "name": "PITR_TARGET",
                            "name": "podinfo"
                        "name": "podinfo",
                        "name": "PODNAME",
                        "name": "postgres"
                            "name": "primary-volume"
                        "name": "primary-volume"
                        "name": "primary-volume",
                    "name": "primary-volume",
                        "name": "REMOVE_BACKUP",
                        "name": "REMOVE_DATA",
                        "name": "REPLICA_NAME",
                        "name": "report",
                        "name": "RESTORE_OPTS",
                    "name": "rmdata",
                            "name": "root-volume"
                        "name": "root-volume"
                        "name": "root-volume",
                    "name": "root-volume",
                    "name": "some-name",
        "name": "some-name",
                    "name": "some-name-backrest-shared-repo",
                "name": "some-name-backrest-shared-repo"
                "name": "some-name-backrest-shared-repo",
            "name": "some-name-backrest-shared-repo",
        "name": "some-name-backrest-shared-repo",
                    "name": "some-name-clone",
        "name": "some-name-clone",
                    "name": "some-name-clone-backrest-shared-repo",
                "name": "some-name-clone-backrest-shared-repo"
                "name": "some-name-clone-backrest-shared-repo",
            "name": "some-name-clone-backrest-shared-repo",
        "name": "some-name-clone-backrest-shared-repo",
                    "name": "some-name-clone-bootstrap",
        "name": "some-name-clone-bootstrap",
                    "name": "some-name-clone-pgbouncer",
                "name": "some-name-clone-pgbouncer",
            "name": "some-name-clone-pgbouncer",
        "name": "some-name-clone-pgbouncer",
                            "name": "some-name-clone-pgbouncer-cm"
                                "name": "some-name-clone-pgbouncer-secret",
                            "name": "some-name-clone-pgbouncer-secret",
        "name": "some-name-clone-pgbr-repo",
                    "name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0"
                                        "name": "some-name-clone-pgha-config",
                                    "name": "some-name-clone-pgha-config",
                               "name": "some-name-clone-pgha-config",
                    "name": "some-name-clone-repl1",
        "name": "some-name-clone-repl1",
                    "name": "some-name-clone-repl2",
        "name": "some-name-clone-repl2",
            "name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl1","pgo-version": "1.6.0"
            "name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl2","pgo-version": "1.6.0"
                "name": "some-name-clone-rmdata-dvik",
        "name": "some-name-clone-rmdata-dvik",
                "name": "some-name-clone-rmdata-jbka",
        "name": "some-name-clone-rmdata-jbka",
                "name": "some-name-clone-rmdata-phhp",
        "name": "some-name-clone-rmdata-phhp",
                "name": "some-name-clone-stanza-create",
                    "name": "some-name-pgbouncer",
                "name": "some-name-pgbouncer",
            "name": "some-name-pgbouncer",
        "name": "some-name-pgbouncer",
                            "name": "some-name-pgbouncer-cm"
                                "name": "some-name-pgbouncer-secret",
                            "name": "some-name-pgbouncer-secret",
        "name": "some-name-pgbr-repo",
                    "name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0"
                                        "name": "some-name-pgha-config",
                               "name": "some-name-pgha-config",
                    "name": "some-name-repl1",
        "name": "some-name-repl1",
                    "name": "some-name-repl2",
        "name": "some-name-repl2",
                "name": "some-name-rmdata-wzoe",
        "name": "some-name-rmdata-wzoe",
                "name": "some-name-rmdata-zqop",
        "name": "some-name-rmdata-zqop",
                "name": "some-name-stanza-create",
                          "name": "ssh-config",
                        "name": "ssh-config",
                      "name": "ssh-config",
                    "name": "ssh-config",
                  "name": "ssh-config",
                            "name": "sshd",
                        "name": "sshd",
                      "name": "sshd",
                    "name": "sshd",
                        "name": "tls-pgbouncer"
                    "name": "tls-pgbouncer",
                            "name": "tls-replication"
                        "name": "tls-replication",
                          "name": "tls-server"
                      "name": "tls-server"
                      "name": "tls-server",
                  "name": "tls-server",
                            "name": "tmp"
                          "name": "tmp"
                        "name": "tmp"
                      "name": "tmp",
                    "name": "tmp",
                  "name": "tmp",
                            "name": "user-volume"
                        "name": "user-volume"
                        "name": "user-volume",
                    "name": "user-volume",
                "nodePort": 0
                                            "operator": "Exists"
                                            "operator": "In",
                                "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh"
                                "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh"
                                        "optional": true
                                    "optional": true
                                    "path": "annotations"
                                    "path": "cpu_limit",
                                    "path": "cpu_request",
                                    "path": "labels"
                                    "path": "mem_limit",
                                    "path": "mem_request",
                                 "path": "ssh_config"
                             "path": "ssh_config"
                           "path": "ssh_config"
                                    "path": "tls-replication.crt"
                                    "path": "tls-replication.key"
                        "periodSeconds": 15,
                    "persistentVolumeClaim": {
                        "persistentVolumeClaim":{"claimName":"some-name"}
                        "persistentVolumeClaim":{"claimName":"some-name-clone"}
                    "persistentVolumeClaim":{"claimName":"some-name-clone"}
                        "persistentVolumeClaim":{"claimName":"some-name-clone-repl1"}
                        "persistentVolumeClaim":{"claimName":"some-name-clone-repl2"}
                        "persistentVolumeClaim":{"claimName":"some-name-repl1"}
                        "persistentVolumeClaim":{"claimName":"some-name-repl2"}
                    "pg-cluster": "some-name"
                    "pg-cluster": "some-name",
                "pg-cluster": "some-name"
                "pg-cluster": "some-name",
            "pg-cluster": "some-name"
            "pg-cluster": "some-name",
                    "pg-cluster": "some-name-clone"
                    "pg-cluster": "some-name-clone",
                "pg-cluster": "some-name-clone"
                "pg-cluster": "some-name-clone",
            "pg-cluster": "some-name-clone"
            "pg-cluster": "some-name-clone",
                    "pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone"
            "pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone"
                    "pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl1","pgo-version": "1.6.0","name": "some-name-clone-replica"
                    "pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl2","pgo-version": "1.6.0","name": "some-name-clone-replica"
            "pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl2","pgo-version": "1.6.0","name": "some-name-clone-replica"
            "pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name","pgo-version": "1.6.0","name": "some-name"
                    "pg-cluster": "some-name","deployment-name": "some-name-repl1","pgo-version": "1.6.0","name": "some-name-replica"
                    "pgha-bootstrap-namespace": "clone-cluster-20561",
            "pgha-bootstrap-namespace": "clone-cluster-20561",
                    "pgha-bootstrap": "some-name-clone",
            "pgha-bootstrap": "some-name-clone",
                    "pgo-backrest-job": "true",
            "pgo-backrest-job": "true",
                    "pgo-backrest-repo": "true"
                "pgo-backrest-repo": "true",
            "pgo-backrest-repo": "true"
                    "pgo-backrest": "true",
                    "pgo-pg-database": "true",
            "pgo-pg-database": "true",
                    "pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone"
            "pgouser": "admin","crunchy-pgha-scope": "some-name-clone","pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone"
                    "pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone"
            "pgo-version": "1.6.0","name": "some-name-clone","pg-cluster": "some-name-clone","deployment-name": "some-name-clone","pgouser": "admin","crunchy-pgha-scope": "some-name-clone"
                    "pgo-version": "1.6.0","name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl1"
            "pgo-version": "1.6.0","name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl1"
                    "pgo-version": "1.6.0","name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl2"
            "pgo-version": "1.6.0","name": "some-name-clone-replica","pg-cluster": "some-name-clone","deployment-name": "some-name-clone-repl2"
                    "pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name"
            "pgo-version": "1.6.0","name": "some-name","pg-cluster": "some-name","deployment-name": "some-name","pgouser": "admin","crunchy-pgha-scope": "some-name"
                    "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1"
            "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl1"
                    "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2"
            "pgo-version": "1.6.0","name": "some-name-replica","pg-cluster": "some-name","deployment-name": "some-name-repl2"
                    "pg-pod-anti-affinity": "preferred",
                "pg-pod-anti-affinity": "preferred",
                    "pgrmdata": "true",
            "pgrmdata": "true",
                            "podAffinityTerm": {
                                        "podAntiAffinity": {
                                    "podAntiAffinity": {
                "podAntiAffinity": {
		                "podAntiAffinity": {
                "port": 2022,
                    "ports": [{
            "ports": [{
                    "preferredDuringSchedulingIgnoredDuringExecution": [
                      "privileged": false,
                        "projected": {
                      "projected": {
                    "projected": {
                  "projected": {
                      "projected": { "sources": [] }
                    "projected": { "sources": [] }
                        "protocol": "TCP",
                "protocol": "TCP",
                    "readinessProbe": {
                            "readOnly": false
                        "readOnly": false
                      "readOnly": false
                      "readOnlyRootFilesystem": true
                            "readOnly": true
                          "readOnly": true
                        "readOnly": true
                      "readOnly": true
            "ReadWriteOnce"
        "replicas": 0,
        "replicas": 1,
        "replicas": 3,
            "requests": {
  "requests": {
                                    "resourceFieldRef": {
                                        "resource": "limits.cpu"
                                        "resource": "limits.memory"
                                        "resource": "requests.cpu"
                                        "resource": "requests.memory"
        "resources": {
"resources": {
                "restartPolicy": "Always",
                "restartPolicy": "Never"
            "rollingUpdate": {
                  "runAsNonRoot": true
                            "secret": {
                          "secret": {
                        "secret": {
                    "secret": {
                  "secret": {
                            "secretKeyRef": {
                            "secretName": "some-name-backrest-repo-config"
                            "secretName": "some-name-backrest-repo-config",
                        "secretName": "some-name-backrest-repo-config"
                      "secretName": "some-name-backrest-repo-config",
                            "secretName": "some-name-clone-backrest-repo-config"
                            "secretName": "some-name-clone-backrest-repo-config",
                        "secretName": "some-name-clone-backrest-repo-config"
                      "secretName": "some-name-clone-backrest-repo-config",
                        "secretName": "some-name-clone-bootstrap-backrest-repo-config"
                        "secretName": "some-name-clone-bootstrap-backrest-repo-config",
                            "secretName": "some-name-clone-postgres-secret"
                        "secretName": "some-name-clone-postgres-secret"
                            "secretName": "some-name-clone-primaryuser-secret"
                        "secretName": "some-name-clone-primaryuser-secret"
                            "secretName": "some-name-clone-some-name-secret"
                        "secretName": "some-name-clone-some-name-secret"
                            "secretName": "some-name-postgres-secret"
                            "secretName": "some-name-primaryuser-secret"
                            "secretName": "some-name-some-name-secret"
                    "securityContext": {
                "securityContext": {
                "securityContext": {},
                "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]},
                "securityContext": {"runAsNonRoot":true,"fsGroup":26},
            "selector": {
        "selector": {
                "serviceAccountName": "pgo-backrest",
                "serviceAccountName": "pgo-default",
                "serviceAccountName": "pgo-pg",
                "serviceAccountName": "pgo-target",
                    "service-name": "some-name-backrest-shared-repo",
            "service-name": "some-name-backrest-shared-repo",
                    "service-name": "some-name-clone-backrest-shared-repo",
            "service-name": "some-name-clone-backrest-shared-repo",
                    "service-name": "some-name-clone-pgbouncer",
                "service-name": "some-name-clone-pgbouncer",
            "service-name": "some-name-clone-pgbouncer",
                    "service-name": "some-name-pgbouncer",
                "service-name": "some-name-pgbouncer",
            "service-name": "some-name-pgbouncer",
            "sessionAffinity": "None"
                    "sizeLimit": "128Ki"
                        "sizeLimit": "16Mi"
                    "sizeLimit": "16Mi"
                      "sizeLimit": "1Mi"
                          "sizeLimit": "2Mi"
                          "sizeLimit": "64Mi"
                    "sizeLimit": "64Mi"
                                                "some-name"
                                                "some-name-clone"
                            "sources": [
                        "sources": [
                      "sources": [
                    "sources": [
            "spec": {
        "spec": {
    "spec": {
                "storage": "1Gi"
        "strategy": {
                "targetPort": 2022,
        "template": {
time="2024-04-23T17:10:08Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.6.0
time="2024-04-23T17:10:48Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:10:53Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:11:45Z" level=error msg="Error completing  workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:11:45Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:12:21Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:12:21Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:13:39Z" level=error msg="get deployment some-name-clone-repl1: deployments.apps \"some-name-clone-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:13:44Z" level=error msg="get deployment some-name-clone-repl2: deployments.apps \"some-name-clone-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:14:11Z" level=error msg="configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0
time="2024-04-23T17:14:11Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0
time="2024-04-23T17:14:56Z" level=error msg="Error completing  workflow some-name-clone-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:14:56Z" level=error msg="pgtasks.pg.percona.com \"some-name-clone-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:15:39Z" level=warning msg="crv1 pgreplica some-name-clone-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:15:39Z" level=warning msg="crv1 pgreplica some-name-clone-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:16:09Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0
time="2024-04-23T17:16:09Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T17:16:09Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0
time="2024-04-23T17:17:08Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-clone-pgha-config (namespace clone-cluster-20561),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0
time="2024-04-23T17:17:08Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-pgha-config (namespace clone-cluster-20561),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0
time="2024-04-23T17:17:15Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:16Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:19Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:35Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:44Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:45Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T17:17:45Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:45Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
time="2024-04-23T17:17:46Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:47Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:47Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:50Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:50Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:51Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:52Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:53Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:54Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:54Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:55Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:17:55Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:01Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:05Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:06Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:08Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-pgha-config (namespace clone-cluster-20561),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0
time="2024-04-23T17:18:10Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:12Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:13Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:18:18Z" level=error msg="jobs.batch \"some-name-clone-rmdata-jbka\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T17:19:51Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:19:52Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:19:56Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:19:57Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:19:59Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:20:00Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:20:01Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:20:06Z" level=error msg="jobs.batch \"some-name-rmdata-zqop\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T17:20:56Z" level=error msg="get deployment some-name-clone-repl1: deployments.apps \"some-name-clone-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:21:01Z" level=error msg="get deployment some-name-clone-repl2: deployments.apps \"some-name-clone-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:21:43Z" level=error msg="configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0
time="2024-04-23T17:21:43Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0
time="2024-04-23T17:22:30Z" level=error msg="Error completing  workflow some-name-clone-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:22:30Z" level=error msg="pgtasks.pg.percona.com \"some-name-clone-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:22:55Z" level=warning msg="crv1 pgreplica some-name-clone-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:22:56Z" level=warning msg="crv1 pgreplica some-name-clone-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:23:09Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.6.0
time="2024-04-23T17:23:09Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T17:23:09Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.6.0
time="2024-04-23T17:23:10Z" level=error msg="-{\"authentication\":{\"replication\":{\"sslcert\":\"/pgconf/tls-replication/tls.crt\",\"sslkey\":\"/pgconf/tls-replication/tls.key\",\"sslmode\":\"verify-ca\",\"sslrootcert\":\"/pgconf/tls/ca.crt\"}},\"callbacks\":{\"on_role_change\":\"/opt/crunchy/bin/postgres-ha/callbacks/pgha-on-role-change.sh\"},\"create_replica_methods\":[\"pgbackrest\",\"basebackup\"],\"pgbackrest\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh replica\",\"keep_data\":true,\"no_params\":true},\"pgbackrest_standby\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh standby\",\"keep_data\":true,\"no_master\":1,\"no_params\":true},\"pgpass\":\"/tmp/.pgpass\",\"remove_data_directory_on_rewind_failure\":true,\"use_unix_socket\":true}\n+{\"authentication\":{\"replication\":{\"sslcert\":\"/pgconf/tls-replication/tls.crt\",\"sslkey\":\"/pgconf/tls-replication/tls.key\",\"sslmode\":\"verify-ca\",\"sslrootcert\":\"/pgconf/tls/ca.crt\"}},\"callbacks\":{\"on_role_change\":\"/opt/crunchy/bin/postgres-ha/callbacks/pgha-on-role-change.sh\"},\"create_replica_methods\":[\"pgbackrest\",\"basebackup\"],\"pg_hba\":[\"local all postgres peer\",\"hostssl replication primaryuser 0.0.0.0/0 cert\",\"hostssl all primaryuser 0.0.0.0/0 reject\",\"hostssl all all 0.0.0.0/0 md5\",\"host replication primaryuser 0.0.0.0/0 md5\",\"host all primaryuser 0.0.0.0/0 reject\",\"host all all 0.0.0.0/0 md5\"],\"pgbackrest\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh replica\",\"keep_data\":true,\"no_params\":true},\"pgbackrest_standby\":{\"command\":\"/opt/crunchy/bin/postgres-ha/pgbackrest/pgbackrest-create-replica.sh standby\",\"keep_data\":true,\"no_master\":1,\"no_params\":true},\"pgpass\":\"/tmp/.pgpass\",\"remove_data_directory_on_rewind_failure\":true,\"use_unix_socket\":true}\n\x1b[0;32mTue Apr 23 17:23:10 UTC 2024 INFO: Reload Config: Detected config change, reloading local configuration\x1b[0m\n\x1b[0;31mTue Apr 23 17:23:10 UTC 2024 ERROR: Error reloading local config: Unable to reload local configuration\x1b[0m\n" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).apply()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:255" version=1.6.0
time="2024-04-23T17:23:10Z" level=error msg="command terminated with exit code 7" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T17:23:10Z" level=error msg="command terminated with exit code 7" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:177" version=1.6.0
time="2024-04-23T17:23:13Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:15Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:16Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:17Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:22Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:26Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:27Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:30Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:32Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:33Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:23:38Z" level=error msg="jobs.batch \"some-name-clone-rmdata-phhp\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T17:24:17Z" level=error msg="get deployment some-name-repl1: deployments.apps \"some-name-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:24:22Z" level=error msg="get deployment some-name-repl2: deployments.apps \"some-name-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:25:05Z" level=error msg="Error completing  workflow some-name-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:25:05Z" level=error msg="pgtasks.pg.percona.com \"some-name-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:25:41Z" level=warning msg="crv1 pgreplica some-name-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:25:41Z" level=warning msg="crv1 pgreplica some-name-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:26:56Z" level=error msg="get deployment some-name-clone-repl1: deployments.apps \"some-name-clone-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:27:01Z" level=error msg="get deployment some-name-clone-repl2: deployments.apps \"some-name-clone-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:27:26Z" level=error msg="configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0
time="2024-04-23T17:27:26Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0
time="2024-04-23T17:28:12Z" level=error msg="Error completing  workflow some-name-clone-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:28:12Z" level=error msg="pgtasks.pg.percona.com \"some-name-clone-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:28:39Z" level=warning msg="crv1 pgreplica some-name-clone-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:28:39Z" level=warning msg="crv1 pgreplica some-name-clone-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:30:08Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-clone-pgha-config (namespace clone-cluster-20561),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0
time="2024-04-23T17:30:08Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-pgha-config (namespace clone-cluster-20561),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.6.0
time="2024-04-23T17:30:17Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:25Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:26Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:27Z" level=error msg="command terminated with exit code 1" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.6.0
time="2024-04-23T17:30:27Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:27Z" level=warning msg="pg_ctl: PID file \"/pgdata/some-name/postmaster.pid\" does not exist\nIs server running?\n" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.applyUpdateToPostgresInstance()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/rolling.go:177" version=1.6.0
time="2024-04-23T17:30:28Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:28Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:29Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:30Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:30Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:31Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:32Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:32Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:33Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:33Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:34Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:35Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:35Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:36Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:37Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:37Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:38Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:41Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:44Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:45Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:47Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:49Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:50Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:51Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:52Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:54Z" level=error msg="pgclusters.pg.percona.com \"some-name-clone\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:54Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:55Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:30:59Z" level=error msg="jobs.batch \"some-name-clone-rmdata-dvik\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T17:31:05Z" level=error msg="pgclusters.pg.percona.com \"some-name\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.6.0
time="2024-04-23T17:31:10Z" level=error msg="jobs.batch \"some-name-rmdata-wzoe\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.6.0
time="2024-04-23T17:31:40Z" level=error msg="get deployment some-name-clone-repl1: deployments.apps \"some-name-clone-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:31:45Z" level=error msg="get deployment some-name-clone-repl2: deployments.apps \"some-name-clone-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.6.0
time="2024-04-23T17:32:29Z" level=error msg="configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.6.0
time="2024-04-23T17:32:29Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-clone-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.6.0
time="2024-04-23T17:33:14Z" level=error msg="Error completing  workflow some-name-clone-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.6.0
time="2024-04-23T17:33:14Z" level=error msg="pgtasks.pg.percona.com \"some-name-clone-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.6.0
time="2024-04-23T17:33:39Z" level=warning msg="crv1 pgreplica some-name-clone-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
time="2024-04-23T17:33:40Z" level=warning msg="crv1 pgreplica some-name-clone-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.6.0
                        "timeoutSeconds": 10
                                "topologyKey": "kubernetes.io/hostname"
            "type": "ClusterIP",
            "type": "RollingUpdate",
                        "value": ""
                        "value": "5432"
  "value": "5432"
                        "value": "/backrestrepo/some-name-backrest-shared-repo"
  "value": "/backrestrepo/some-name-backrest-shared-repo"
                        "value": "/backrestrepo/some-name-clone-backrest-shared-repo"
  "value": "/backrestrepo/some-name-clone-backrest-shared-repo"
                        "value": "backup"
                        "value": "crunchy-pgha-scope"
                        "value": "db"
  "value": "db"
                        "value": " --db-host=10.55.249.51 --db-path=/pgdata/some-name"
                        "value": " --db-host=10.55.249.63 --db-path=/pgdata/some-name-clone"
                        "value": " --db-host=10.55.249.66 --db-path=/pgdata/some-name"
                        "value": " --db-host=10.55.249.75 --db-path=/pgdata/some-name-clone"
                        "value": " --db-host=10.55.250.123 --db-path=/pgdata/some-name-clone"
                        "value": " --db-host=10.55.250.135 --db-path=/pgdata/some-name-clone"
                        "value": "false"
  "value": "false"
                        "valueFrom": {
                        "value": "INFO"
                        "value": "pgbackrest"
                        "value": "pgbackrest_init"
                        "value": "pgbackrest-repo"
                        "value": "/pgdata/some-name"
  "value": "/pgdata/some-name"
                        "value": "/pgdata/some-name-clone"
  "value": "/pgdata/some-name-clone"
                        "value": "/pgdata/some-name-clone-repl1"
  "value": "/pgdata/some-name-clone-repl1"
                        "value": "/pgdata/some-name-clone-repl2"
  "value": "/pgdata/some-name-clone-repl2"
                        "value": "/pgdata/some-name-repl1"
  "value": "/pgdata/some-name-repl1"
                        "value": "/pgdata/some-name-repl2"
  "value": "/pgdata/some-name-repl2"
                        "value": "posix"
  "value": "posix"
                        "value": "postgres"
                                            "values": [
                        "value": "some-name"
  "value": "some-name-backrest-shared-repo"
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-c2lt7"
                        "value": "some-name-backrest-shared-repo-6f8f947dbc-c79zl"
  "value": "some-name-backrest-shared-repo.clone-cluster-20561.svc.cluster.local."
                        "value": "some-name-clone"
  "value": "some-name-clone-backrest-shared-repo"
                        "value": "some-name-clone-backrest-shared-repo-64f687d86c-4pl79"
                        "value": "some-name-clone-backrest-shared-repo-64f687d86c-6t4gx"
                        "value": "some-name-clone-backrest-shared-repo-64f687d86c-g7nsk"
                        "value": "some-name-clone-backrest-shared-repo-64f687d86c-nmbwm"
                        "value": "stanza-create"
                        "value": "/tmp"
  "value": "/tmp"
                        "value": "/tmp/nss_wrapper/pgbackrest-repo/group"
                        "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd"
                      "value": "/tmp/nss_wrapper/postgres/group"
                      "value": "/tmp/nss_wrapper/postgres/passwd"
                        "value": "true"
  "value": "true"
                        "value": "--type=full --db-host=10.55.249.51 --db-path=/pgdata/some-name"
                        "value": "--type=full --db-host=10.55.249.63 --db-path=/pgdata/some-name-clone"
                        "value": "--type=full --db-host=10.55.249.66 --db-path=/pgdata/some-name"
                        "value": "--type=full --db-host=10.55.249.75 --db-path=/pgdata/some-name-clone"
                        "value": "--type=full --db-host=10.55.250.123 --db-path=/pgdata/some-name-clone"
                        "value": "--type=full --db-host=10.55.250.135 --db-path=/pgdata/some-name-clone"
                        "value": "/usr/lib64/libnss_wrapper.so"
                      "value": "/usr/lib64/libnss_wrapper.so"
                        "value": "{vendor: \"crunchydata\"}"
                    "vendor": "crunchydata"
                    "vendor": "crunchydata",
                "vendor": "crunchydata"
                "vendor": "crunchydata",
            "vendor": "crunchydata"
            "vendor": "crunchydata",
                    "volumeMounts": [
                    "volumeMounts": [{
                "volumes": [
                "volumes": [{
                            "weight": 1,
+ destroy_operator
+ kubectl_bin delete -f /tmp/tmp.UloFqeshVN/operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.yIJRo4CDKl
++ mktemp
+ local LAST_ERR=/tmp/tmp.j7m3LYHtPo
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.UloFqeshVN/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.UloFqeshVN/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.UloFqeshVN/operator.yaml
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.yIJRo4CDKl
+ cat /tmp/tmp.j7m3LYHtPo
Error from server (NotFound): error when deleting "/tmp/tmp.UloFqeshVN/operator.yaml": serviceaccounts "pgo-deployer-sa" not found
Error from server (NotFound): error when deleting "/tmp/tmp.UloFqeshVN/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
Error from server (NotFound): error when deleting "/tmp/tmp.UloFqeshVN/operator.yaml": configmaps "pgo-deployer-cm" not found
Error from server (NotFound): error when deleting "/tmp/tmp.UloFqeshVN/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found
Error from server (NotFound): error when deleting "/tmp/tmp.UloFqeshVN/operator.yaml": jobs.batch "pgo-deploy" not found
+ rm /tmp/tmp.yIJRo4CDKl /tmp/tmp.j7m3LYHtPo
+ return 1
+ true
+ yq w -d4 /tmp/tmp.UloFqeshVN/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall
+ kubectl_bin apply -f /tmp/tmp.UloFqeshVN/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.ncaj55gWqE
++ mktemp
+ local LAST_ERR=/tmp/tmp.X3biBhfj8G
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl apply -f /tmp/tmp.UloFqeshVN/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.ncaj55gWqE
serviceaccount/pgo-deployer-sa created
clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created
configmap/pgo-deployer-cm created
clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created
job.batch/pgo-deploy created
+ cat /tmp/tmp.X3biBhfj8G
+ rm /tmp/tmp.ncaj55gWqE /tmp/tmp.X3biBhfj8G
+ return 0
+ wait_job_completion pgo-deploy true
+ local job=pgo-deploy
+ local mandatory=true
+ local ns=clone-cluster-20561
+ retry=0
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
pgo-deploy+ retry=0
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 1 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 2 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 3 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 4 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 5 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 6 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 7 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 8 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 9 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 10 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 11 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 12 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 13 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ '' == \1 ]]
+ sleep 1
+ echo -n .
.+ let retry+=1
+ '[' 14 -ge 600 ']'
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.metadata.name}'
++ grep -io 'not found'
+ [[ '' == \n\o\t\ \f\o\u\n\d ]]
++ kubectl_bin -n clone-cluster-20561 get job/pgo-deploy -o 'jsonpath={.status.succeeded}'
+ [[ 1 == \1 ]]
+ echo

+ kubectl_bin delete -f /tmp/tmp.UloFqeshVN/operator-destroy.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.2XAhchFZji
++ mktemp
+ local LAST_ERR=/tmp/tmp.bDIFT857uw
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete -f /tmp/tmp.UloFqeshVN/operator-destroy.yaml
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.2XAhchFZji
serviceaccount "pgo-deployer-sa" deleted
clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted
configmap "pgo-deployer-cm" deleted
clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted
job.batch "pgo-deploy" deleted
+ cat /tmp/tmp.bDIFT857uw
+ rm /tmp/tmp.2XAhchFZji /tmp/tmp.bDIFT857uw
+ return 0
+ kubectl_bin get ns
+ egrep -v '^kube-|^default|Terminating|openshift|^NAME'
+ awk '{print$1}'
++ mktemp
+ xargs kubectl delete ns --grace-period=0 --force=true
+ local LAST_OUT=/tmp/tmp.cX52sDyZnn
++ mktemp
+ local LAST_ERR=/tmp/tmp.4QnLnN5KlV
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl get ns
+ exit_status=0
+ [[ 0 != 0 ]]
+ break
+ set -o errexit
+ cat /tmp/tmp.cX52sDyZnn
+ cat /tmp/tmp.4QnLnN5KlV
+ rm /tmp/tmp.cX52sDyZnn /tmp/tmp.4QnLnN5KlV
+ return 0
Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
namespace "clone-cluster-20561" force deleted
+ kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.6TJTmh84iN
++ mktemp
+ local LAST_ERR=/tmp/tmp.PkHqYvbFja
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.6TJTmh84iN
+ cat /tmp/tmp.PkHqYvbFja
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.6TJTmh84iN /tmp/tmp.PkHqYvbFja
+ return 1
+ true
+ kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr
++ mktemp
+ local LAST_OUT=/tmp/tmp.nJMPkPQjHR
++ mktemp
+ local LAST_ERR=/tmp/tmp.9jg58cyqsl
+ local exit_status=0
+ local timeout=4
+ set +o errexit
++ seq 0 2
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 0
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 4
+ for i in '$(seq 0 2)'
+ KUBECONFIG=/tmp/jen-pg-721-a67b3475b-2-sandbox
+ kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr
+ exit_status=1
+ [[ 1 != 0 ]]
+ sleep 8
+ set -o errexit
+ cat /tmp/tmp.nJMPkPQjHR
+ cat /tmp/tmp.9jg58cyqsl
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found
Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found
+ rm /tmp/tmp.nJMPkPQjHR /tmp/tmp.9jg58cyqsl
+ return 1
+ true
+ rm -rf /tmp/tmp.UloFqeshVN