++ echo 'Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/scheduled-backup.log' Log: /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/logs/scheduled-backup.log ++ '[' -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: the server doesn't have a resource type "projects" +++ kubectl version -o json +++ jq -r .serverVersion.gitVersion +++ grep '\-eks\-' WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ '[' ']' +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ /usr/bin/sed -r 's/[^0-9.]+//g' +++ kubectl version -o json WARNING: version difference between client (1.30) and server (1.26) exceeds the supported minor version skew of +/-1 ++ KUBE_VERSION=1.26 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.15.1 ++ '[' v3 == v2 ']' + main + create_namespace scheduled-backup-13036 + local namespace=scheduled-backup-13036 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + awk '{print$1}' + cleanup_rbac + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + xargs kubectl delete ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.7xGKkkdqOs + local LAST_OUT=/tmp/tmp.qHylvjpAoe ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.9ZowkYEQ8V + local exit_status=0 + local timeout=4 + set +o errexit + local LAST_ERR=/tmp/tmp.RMYcIM5pp0 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qHylvjpAoe + cat /tmp/tmp.RMYcIM5pp0 E0524 17:37:05.471915 32027 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:05.702594 32027 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:05.809358 32027 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:05.916161 32027 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request + rm /tmp/tmp.qHylvjpAoe /tmp/tmp.RMYcIM5pp0 + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level E0524 17:37:07.292951 32195 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:07.541585 32195 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:07.647508 32195 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:07.753171 32195 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request error: resource(s) were provided, but no name was specified + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.7xGKkkdqOs + cat /tmp/tmp.9ZowkYEQ8V E0524 17:37:13.702464 32736 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:13.973707 32736 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:14.108519 32736 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:14.214892 32736 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.7xGKkkdqOs /tmp/tmp.9ZowkYEQ8V + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level ++ mktemp + local LAST_OUT=/tmp/tmp.5npSYoZUD0 ++ mktemp + local LAST_ERR=/tmp/tmp.qPo0dA5EhC + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-controller-manager-cluster-level + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.5npSYoZUD0 + cat /tmp/tmp.qPo0dA5EhC E0524 17:37:31.536510 906 memcache.go:287] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:31.749751 906 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:31.856447 906 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request E0524 17:37:31.963257 906 memcache.go:121] couldn't get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + rm /tmp/tmp.5npSYoZUD0 /tmp/tmp.qPo0dA5EhC + return 1 + true + kubectl delete MutatingWebhookConfiguration/chaos-mesh-mutation ValidatingWebhookConfiguration/chaos-mesh-validation ValidatingWebhookConfiguration/validate-auth Error from server (NotFound): mutatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-mutation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "chaos-mesh-validation" not found Error from server (NotFound): validatingwebhookconfigurations.admissionregistration.k8s.io "validate-auth" not found + true + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + kubectl_bin delete namespace scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.eg7ck0xmvl ++ mktemp + local LAST_ERR=/tmp/tmp.Hb8R1Csb0K + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete namespace scheduled-backup-13036 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete namespace scheduled-backup-13036 + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete namespace scheduled-backup-13036 + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.eg7ck0xmvl + cat /tmp/tmp.Hb8R1Csb0K Error from server (NotFound): namespaces "scheduled-backup-13036" not found + rm /tmp/tmp.eg7ck0xmvl /tmp/tmp.Hb8R1Csb0K + return 1 + : + wait_for_delete namespace/scheduled-backup-13036 + local res=namespace/scheduled-backup-13036 + set +o xtrace namespace/scheduled-backup-13036 - Error from server (NotFound): namespaces "scheduled-backup-13036" not found + kubectl_bin create namespace scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.2qTftq9vkd ++ mktemp + local LAST_ERR=/tmp/tmp.4UjxZ3p7vl + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl create namespace scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2qTftq9vkd namespace/scheduled-backup-13036 created + cat /tmp/tmp.4UjxZ3p7vl + rm /tmp/tmp.2qTftq9vkd /tmp/tmp.4UjxZ3p7vl + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.qUzM5xp9Bt +++ mktemp ++ local LAST_ERR=/tmp/tmp.xGv4KOcxrN ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.qUzM5xp9Bt ++ cat /tmp/tmp.xGv4KOcxrN ++ rm /tmp/tmp.qUzM5xp9Bt /tmp/tmp.xGv4KOcxrN ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-scheduled-backup --namespace=scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.pB1gN8jJc2 ++ mktemp + local LAST_ERR=/tmp/tmp.XRj2vTOHWn + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl config set-context gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-scheduled-backup --namespace=scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.pB1gN8jJc2 Context "gke_cloud-dev-112233_us-central1-c_jen-pg-776-0087ef129-2-scheduled-backup" modified. + cat /tmp/tmp.XRj2vTOHWn + rm /tmp/tmp.pB1gN8jJc2 /tmp/tmp.XRj2vTOHWn + return 0 + deploy_operator + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local action=install + local namespace=scheduled-backup-13036 + local dont_send_telemetry=false + local version_service_uri_override= + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + prepare_operator_yaml '' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml disabled install scheduled-backup-13036 false + local pull_secret_name= + local operator_manifest=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml + local namespace_mode=disabled + local operator_action=install + local namespace=scheduled-backup-13036 + local dont_send_telemetry=false + /usr/bin/sed -e 's#^namespace: .*#namespace: "scheduled-backup-13036"#g' + /usr/bin/sed -e 's#namespace_mode: .*#namespace_mode: "disabled"#g' + /usr/bin/sed -e 's#^disable_telemetry: .*#disable_telemetry: "false"#g' + /usr/bin/sed -e 's#pgo_operator_namespace: .*#pgo_operator_namespace: "scheduled-backup-13036"#g' + yq r -d2 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml 'data[values.yaml]' ++ cut -d: -f1 ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 ++ echo perconalab/percona-postgresql-operator:PR-776-0087ef129 + /usr/bin/sed -e 's#pgo_image_prefix: .*#pgo_image_prefix: "perconalab/percona-postgresql-operator"#g' ++ cut -d: -f2 + /usr/bin/sed -e 's#pgo_image_tag: .*#pgo_image_tag: "PR-776-0087ef129"#g' + cat /dev/fd/63 - ++ echo 'crunchy_debug: "true"' + [[ -n '' ]] + [[ GKE == \O\P\E\N\S\H\I\F\T ]] + yq w -d3 - 'subjects[0].namespace' scheduled-backup-13036 + yq w -d4 - 'spec.template.spec.containers[0].image' perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-deployer + yq w '-d*' /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/operator.yaml metadata.namespace scheduled-backup-13036 + yq w -d4 - 'spec.template.spec.containers[0].env[0].value' install ++ cat /tmp/tmp.7cfNRxU4oI/operator.ini + yq w -d2 - -d2 'data[values.yaml]' 'crunchy_debug: "true" archive_mode: "true" archive_timeout: "60" ccp_image_pull_secret: "" ccp_image_pull_secret_manifest: "" create_rbac: "true" delete_operator_namespace: "false" delete_watched_namespaces: "false" disable_telemetry: "false" namespace: "scheduled-backup-13036" namespace_mode: "disabled" pgo_image_prefix: "perconalab/percona-postgresql-operator" pgo_image_pull_policy: "Always" pgo_image_pull_secret: "" pgo_image_pull_secret_manifest: "" pgo_image_tag: "PR-776-0087ef129" pgo_operator_namespace: "scheduled-backup-13036"' + [[ -n '' ]] + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.43DAbW9oyL ++ mktemp + local LAST_ERR=/tmp/tmp.h9ajI8BPau + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.43DAbW9oyL serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.h9ajI8BPau + rm /tmp/tmp.43DAbW9oyL /tmp/tmp.h9ajI8BPau + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.P2X0Hj12ZZ ++ mktemp + local LAST_ERR=/tmp/tmp.j0KBj4CjGg + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.P2X0Hj12ZZ serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.j0KBj4CjGg + rm /tmp/tmp.P2X0Hj12ZZ /tmp/tmp.j0KBj4CjGg + return 0 ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rfgv6sMS4J +++ mktemp ++ local LAST_ERR=/tmp/tmp.z8HqhxC2Tu ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.rfgv6sMS4J ++ cat /tmp/tmp.z8HqhxC2Tu ++ rm /tmp/tmp.rfgv6sMS4J /tmp/tmp.z8HqhxC2Tu ++ return 0 + wait_pod postgres-operator-57d977b5cc-hssbr + local pod=postgres-operator-57d977b5cc-hssbr + local ns= + set +o xtrace postgres-operator-57d977b5cc-hssbrtrue + [[ -n '' ]] + cluster=some-name-11182 + desc 'Testing full backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing full backup/restore from s3 ----------------------------------------------------------------------------------- + apply_configs some-name-11182 + local cluster=some-name-11182 + set +x secret/some-name-11182-backrest-repo-config created + yq w - metadata.name some-name-11182-custom-config + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/custom-config.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Z663Yv2y9n ++ mktemp + local LAST_ERR=/tmp/tmp.PckBXsp7Zc + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Z663Yv2y9n configmap/some-name-11182-custom-config created + cat /tmp/tmp.PckBXsp7Zc + rm /tmp/tmp.Z663Yv2y9n /tmp/tmp.PckBXsp7Zc + return 0 + spinup_pgcluster some-name-11182 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml s3 some-name-11182-custom-config false true full + local cluster=some-name-11182 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-11182-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=scheduled-backup-13036 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-11182 scheduled-backup-13036 + local cluster=some-name-11182 + local namespace=scheduled-backup-13036 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - metadata.name some-name-11182-users-secret + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + yq w - stringData.some-name some-name_pass + local LAST_OUT=/tmp/tmp.uZI4hL3RGn ++ mktemp + local LAST_ERR=/tmp/tmp.mmXGszTPle + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.uZI4hL3RGn secret/some-name-11182-users-secret created + cat /tmp/tmp.mmXGszTPle + rm /tmp/tmp.uZI4hL3RGn /tmp/tmp.mmXGszTPle + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - metadata.name some-name-11182-pmm-secret + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.o82bFgBeX3 ++ mktemp + local LAST_ERR=/tmp/tmp.h3W4Y8QcK9 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.o82bFgBeX3 secret/some-name-11182-pmm-secret created + cat /tmp/tmp.h3W4Y8QcK9 + rm /tmp/tmp.o82bFgBeX3 /tmp/tmp.h3W4Y8QcK9 + return 0 + kubectl_bin -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.MOJ6ZizSNw ++ mktemp + local LAST_ERR=/tmp/tmp.QX0ZaU4vXE + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.MOJ6ZizSNw deployment.apps/pg-client created + cat /tmp/tmp.QX0ZaU4vXE + rm /tmp/tmp.MOJ6ZizSNw /tmp/tmp.QX0ZaU4vXE + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name-11182 s3 false false some-name-11182-custom-config false true full false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name-11182 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-11182-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local root_via_pgbouncer= + yq w - metadata.labels.crunchy-pgha-scope some-name-11182 + yq w - metadata.labels.deployment-name some-name-11182 + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name-11182 + yq w - spec.keepBackups false + yq w - metadata.name some-name-11182 + yq w - spec.secretsName some-name-11182-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.user some-name + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - spec.keepData false + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - metadata.labels.pg-cluster some-name-11182 + yq w - spec.name some-name-11182 + yq w - spec.database some-name-11182 + yq w - metadata.labels.name some-name-11182 + yq w - spec.clustername some-name-11182 + case ${podAntiAffinity} in + case ${backup} in + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + [[ true == \t\r\u\e ]] + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.schedule[0].type' full + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + mv /tmp/tmp.7cfNRxU4oI/cr.backup.yaml /tmp/tmp.7cfNRxU4oI/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-11182-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.7cfNRxU4oI/cr.yaml spec.pgPrimary.customconfig some-name-11182-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.5nSAP6nb1r ++ mktemp + local LAST_ERR=/tmp/tmp.MdgV8oEfEG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.5nSAP6nb1r perconapgcluster.pg.percona.com/some-name-11182 created + cat /tmp/tmp.MdgV8oEfEG + rm /tmp/tmp.5nSAP6nb1r /tmp/tmp.MdgV8oEfEG + return 0 + wait_deployment some-name-11182-backrest-shared-repo + local name=some-name-11182-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-11182-backrest-shared-repo.... + wait_deployment some-name-11182 + local name=some-name-11182 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-11182........... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-11182-stanza-create false + local job=some-name-11182-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' some-name-11182-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-11182 + local job=backrest-backup-some-name-11182 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-11182+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-11182 + cluster_name=some-name-11182 + wait_cluster_status some-name-11182 'pgcluster Initialized' + local cluster_name=some-name-11182 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-13036 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.za9nsE9rit +++ mktemp ++ local LAST_ERR=/tmp/tmp.oLbr5MehPP ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.za9nsE9rit ++ cat /tmp/tmp.oLbr5MehPP ++ rm /tmp/tmp.za9nsE9rit /tmp/tmp.oLbr5MehPP ++ return 0 + status_query='kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lk7kdqbSdT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Mcz1DCRUZh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lk7kdqbSdT +++ cat /tmp/tmp.Mcz1DCRUZh +++ rm /tmp/tmp.lk7kdqbSdT /tmp/tmp.Mcz1DCRUZh +++ return 0 + [[ pgcluster Processed == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + let retry+=1 + '[' 1 -ge 24 ']' + echo 'waiting for cluster readyness' waiting for cluster readyness + sleep 10 ++ eval kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uJmGk6Axs2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.XxN0xSVJHm +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uJmGk6Axs2 +++ cat /tmp/tmp.XxN0xSVJHm +++ rm /tmp/tmp.uJmGk6Axs2 /tmp/tmp.XxN0xSVJHm +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6oO8NHAdy3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zVqVSkswHN +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6oO8NHAdy3 +++ cat /tmp/tmp.zVqVSkswHN +++ rm /tmp/tmp.6oO8NHAdy3 /tmp/tmp.zVqVSkswHN +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.4KdgouEBBV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6IstR5qPKn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.4KdgouEBBV +++ cat /tmp/tmp.6IstR5qPKn +++ rm /tmp/tmp.4KdgouEBBV /tmp/tmp.6IstR5qPKn +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rVEVyC2EuJ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.W4lsY9kWwp +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.rVEVyC2EuJ +++ cat /tmp/tmp.W4lsY9kWwp +++ rm /tmp/tmp.rVEVyC2EuJ /tmp/tmp.W4lsY9kWwp +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.MkJbGoA2ho +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.p8LW2Vkq05 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.MkJbGoA2ho ++++ cat /tmp/tmp.p8LW2Vkq05 ++++ rm /tmp/tmp.MkJbGoA2ho /tmp/tmp.p8LW2Vkq05 ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zvWdjvNYX6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.J0cbTU8SLV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zvWdjvNYX6 +++ cat /tmp/tmp.J0cbTU8SLV +++ rm /tmp/tmp.zvWdjvNYX6 /tmp/tmp.J0cbTU8SLV +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vrY5gndzen +++ mktemp ++ local LAST_ERR=/tmp/tmp.LclKTER2ve ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.vrY5gndzen ++ cat /tmp/tmp.LclKTER2ve ++ rm /tmp/tmp.vrY5gndzen /tmp/tmp.LclKTER2ve ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.l8OzMHtqRl ++ mktemp + local LAST_ERR=/tmp/tmp.vpkBubtO8N + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.l8OzMHtqRl + cat /tmp/tmp.vpkBubtO8N + rm /tmp/tmp.l8OzMHtqRl /tmp/tmp.vpkBubtO8N + return 0 ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Rkpop5daA9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bndpQV6Rhi +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Rkpop5daA9 +++ cat /tmp/tmp.bndpQV6Rhi +++ rm /tmp/tmp.Rkpop5daA9 /tmp/tmp.bndpQV6Rhi +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.k6XNQtf57X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.HVeGXFzYnc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.k6XNQtf57X +++ cat /tmp/tmp.HVeGXFzYnc +++ rm /tmp/tmp.k6XNQtf57X /tmp/tmp.HVeGXFzYnc +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.MN4UWCPSgb ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5G9yxFuun2 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.MN4UWCPSgb +++ cat /tmp/tmp.5G9yxFuun2 +++ rm /tmp/tmp.MN4UWCPSgb /tmp/tmp.5G9yxFuun2 +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.RmE8u7Saqt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zzTcuGY9tg ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.RmE8u7Saqt ++++ cat /tmp/tmp.zzTcuGY9tg ++++ rm /tmp/tmp.RmE8u7Saqt /tmp/tmp.zzTcuGY9tg ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xHMkI9xGq3 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gFmLSOMAef +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.xHMkI9xGq3 +++ cat /tmp/tmp.gFmLSOMAef +++ rm /tmp/tmp.xHMkI9xGq3 /tmp/tmp.gFmLSOMAef +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.LYg93X5TyO +++ mktemp ++ local LAST_ERR=/tmp/tmp.PDEkUYxS4N ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.LYg93X5TyO ++ cat /tmp/tmp.PDEkUYxS4N ++ rm /tmp/tmp.LYg93X5TyO /tmp/tmp.PDEkUYxS4N ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.v5JsDYuYG2 ++ mktemp + local LAST_ERR=/tmp/tmp.4p51rFupGw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.v5JsDYuYG2 + cat /tmp/tmp.4p51rFupGw + rm /tmp/tmp.v5JsDYuYG2 /tmp/tmp.4p51rFupGw + return 0 ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ULmsbPHw92 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.uZu3s9nyTp +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ULmsbPHw92 +++ cat /tmp/tmp.uZu3s9nyTp +++ rm /tmp/tmp.ULmsbPHw92 /tmp/tmp.uZu3s9nyTp +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.43RISPjB9S ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NrxwPAQjzp +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.43RISPjB9S +++ cat /tmp/tmp.NrxwPAQjzp +++ rm /tmp/tmp.43RISPjB9S /tmp/tmp.NrxwPAQjzp +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OhR9O8cIh7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SAaWGmBVjk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.OhR9O8cIh7 +++ cat /tmp/tmp.SAaWGmBVjk +++ rm /tmp/tmp.OhR9O8cIh7 /tmp/tmp.SAaWGmBVjk +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.niLggdPaZU +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.ESFRCoVqHX ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.niLggdPaZU ++++ cat /tmp/tmp.ESFRCoVqHX ++++ rm /tmp/tmp.niLggdPaZU /tmp/tmp.ESFRCoVqHX ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6ZO79JE4uQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WaF8Nmi2kJ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6ZO79JE4uQ +++ cat /tmp/tmp.WaF8Nmi2kJ +++ rm /tmp/tmp.6ZO79JE4uQ /tmp/tmp.WaF8Nmi2kJ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.4QPItWmskh +++ mktemp ++ local LAST_ERR=/tmp/tmp.VYKQ8IYYVQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.4QPItWmskh ++ cat /tmp/tmp.VYKQ8IYYVQ ++ rm /tmp/tmp.4QPItWmskh /tmp/tmp.VYKQ8IYYVQ ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.27XvZOYjKC ++ mktemp + local LAST_ERR=/tmp/tmp.mFzFUFo4wv + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.27XvZOYjKC + cat /tmp/tmp.mFzFUFo4wv + rm /tmp/tmp.27XvZOYjKC /tmp/tmp.mFzFUFo4wv + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.USsqqiYYYd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Uka7FjYnFk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.USsqqiYYYd +++ cat /tmp/tmp.Uka7FjYnFk +++ rm /tmp/tmp.USsqqiYYYd /tmp/tmp.Uka7FjYnFk +++ return 0 ++ next_backup_after_sec some-name-11182-sch-backup ++ local conf_map_name=some-name-11182-sch-backup ++++ kubectl_bin get configmap/some-name-11182-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.8YIe2UzoWQ +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.fl7UKyZfuD ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get configmap/some-name-11182-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.8YIe2UzoWQ ++++ cat /tmp/tmp.fl7UKyZfuD ++++ rm /tmp/tmp.8YIe2UzoWQ /tmp/tmp.fl7UKyZfuD ++++ return 0 +++ /usr/bin/date +%s --date=2024-05-24T17:40:01Z --utc ++ local configmap_creation_time=1716572401 +++ kubectl_bin get configmap/some-name-11182-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Gs0YDDoZdN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.nfy8fls7st +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get configmap/some-name-11182-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Gs0YDDoZdN +++ cat /tmp/tmp.nfy8fls7st +++ rm /tmp/tmp.Gs0YDDoZdN /tmp/tmp.nfy8fls7st +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1716572401 300 +++ local start_time_sec=1716572401 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716572701 +++ local next_interval_min_edge=1 +++ local next_backup_at_sec=1716572700 +++ echo 1716572700 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 17:44:27 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 17:44:27 GMT ' --utc +%s +++ echo 1716572667 ++ local result=33 ++ [[ 33 -gt 0 ]] ++ echo 33 + sleep 33 + kubectl_bin wait job/some-name-11182-full-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.47y8BcS4Xp ++ mktemp + local LAST_ERR=/tmp/tmp.wEwxWaz40i + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl wait job/some-name-11182-full-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.47y8BcS4Xp job.batch/some-name-11182-full-sch-backup condition met + cat /tmp/tmp.wEwxWaz40i + rm /tmp/tmp.47y8BcS4Xp /tmp/tmp.wEwxWaz40i + return 0 + restore some-name-11182 s3 full + local cluster=some-name-11182 + local storage=s3 + local backup_type=full + local restore_target= ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.0caXMTNls5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hUdYsP6hiE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.0caXMTNls5 +++ cat /tmp/tmp.hUdYsP6hiE +++ rm /tmp/tmp.0caXMTNls5 /tmp/tmp.hUdYsP6hiE +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.egZ9eFJ6YV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.H9Uqy0VV48 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.egZ9eFJ6YV +++ cat /tmp/tmp.H9Uqy0VV48 +++ rm /tmp/tmp.egZ9eFJ6YV /tmp/tmp.H9Uqy0VV48 +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dH5ZisCimq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.hgvHNUhIJS +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.dH5ZisCimq +++ cat /tmp/tmp.hgvHNUhIJS +++ rm /tmp/tmp.dH5ZisCimq /tmp/tmp.hgvHNUhIJS +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zZboNhBGae +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RzWwRzkVPo ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zZboNhBGae ++++ cat /tmp/tmp.RzWwRzkVPo ++++ rm /tmp/tmp.zZboNhBGae /tmp/tmp.RzWwRzkVPo ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.14noHqJcBz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.H96I8U2tNW +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.14noHqJcBz +++ cat /tmp/tmp.H96I8U2tNW +++ rm /tmp/tmp.14noHqJcBz /tmp/tmp.H96I8U2tNW +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FigTQmsers +++ mktemp ++ local LAST_ERR=/tmp/tmp.yR5FJa9Nh9 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FigTQmsers ++ cat /tmp/tmp.yR5FJa9Nh9 ++ rm /tmp/tmp.FigTQmsers /tmp/tmp.yR5FJa9Nh9 ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.XMGdjAbRZS ++ mktemp + local LAST_ERR=/tmp/tmp.RiW4agsG1o + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.XMGdjAbRZS + cat /tmp/tmp.RiW4agsG1o + rm /tmp/tmp.XMGdjAbRZS /tmp/tmp.RiW4agsG1o + return 0 ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NhzKB61chd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.K1XdR16Kvu +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NhzKB61chd +++ cat /tmp/tmp.K1XdR16Kvu +++ rm /tmp/tmp.NhzKB61chd /tmp/tmp.K1XdR16Kvu +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.muhpocj3ks ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SRmhJa8jjv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.muhpocj3ks +++ cat /tmp/tmp.SRmhJa8jjv +++ rm /tmp/tmp.muhpocj3ks /tmp/tmp.SRmhJa8jjv +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.IRWxLC4obn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.rSkBHofTmm +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.IRWxLC4obn +++ cat /tmp/tmp.rSkBHofTmm +++ rm /tmp/tmp.IRWxLC4obn /tmp/tmp.rSkBHofTmm +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.TpAWrnnG03 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.DmmpYXUemy ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.TpAWrnnG03 ++++ cat /tmp/tmp.DmmpYXUemy ++++ rm /tmp/tmp.TpAWrnnG03 /tmp/tmp.DmmpYXUemy ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XKUkP1VBfA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.517iU4yH4b +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XKUkP1VBfA +++ cat /tmp/tmp.517iU4yH4b +++ rm /tmp/tmp.XKUkP1VBfA /tmp/tmp.517iU4yH4b +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-11182.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.zCKtasBg2P +++ mktemp ++ local LAST_ERR=/tmp/tmp.ZGTJXemll1 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.zCKtasBg2P ++ cat /tmp/tmp.ZGTJXemll1 ++ rm /tmp/tmp.zCKtasBg2P /tmp/tmp.ZGTJXemll1 ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.6w9hW991qR ++ mktemp + local LAST_ERR=/tmp/tmp.yBUXr1UxJG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.6w9hW991qR + cat /tmp/tmp.yBUXr1UxJG + rm /tmp/tmp.6w9hW991qR /tmp/tmp.yBUXr1UxJG + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.I5Juw4HdEm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UV1rBscPhE +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.I5Juw4HdEm +++ cat /tmp/tmp.UV1rBscPhE +++ rm /tmp/tmp.I5Juw4HdEm /tmp/tmp.UV1rBscPhE +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HprkiaieoR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ROZF6OhWqm +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.HprkiaieoR +++ cat /tmp/tmp.ROZF6OhWqm +++ rm /tmp/tmp.HprkiaieoR /tmp/tmp.ROZF6OhWqm +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ME5NagH3Ia ++++ mktemp +++ local LAST_ERR=/tmp/tmp.P9YdgPVdKd +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ME5NagH3Ia +++ cat /tmp/tmp.P9YdgPVdKd +++ rm /tmp/tmp.ME5NagH3Ia /tmp/tmp.P9YdgPVdKd +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.UCnQg2xc39 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.vHmCUxGF4A ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.UCnQg2xc39 ++++ cat /tmp/tmp.vHmCUxGF4A ++++ rm /tmp/tmp.UCnQg2xc39 /tmp/tmp.vHmCUxGF4A ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uu7WJdtOAm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.dAkd4R35sc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uu7WJdtOAm +++ cat /tmp/tmp.dAkd4R35sc +++ rm /tmp/tmp.uu7WJdtOAm /tmp/tmp.dAkd4R35sc +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.uk6D47HfBv +++ mktemp ++ local LAST_ERR=/tmp/tmp.eO0SItq8nm ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.uk6D47HfBv ++ cat /tmp/tmp.eO0SItq8nm ++ rm /tmp/tmp.uk6D47HfBv /tmp/tmp.eO0SItq8nm ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.mAemLFRRwd ++ mktemp + local LAST_ERR=/tmp/tmp.YuSpUddqaD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.mAemLFRRwd + cat /tmp/tmp.YuSpUddqaD + rm /tmp/tmp.mAemLFRRwd /tmp/tmp.YuSpUddqaD + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-11182 +++ local user=postgres +++ local cluster=some-name-11182 +++ local namespace=scheduled-backup-13036 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.O2L5cDCco3 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.XiOfDAHLyp ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.O2L5cDCco3 ++++ cat /tmp/tmp.XiOfDAHLyp ++++ rm /tmp/tmp.O2L5cDCco3 /tmp/tmp.XiOfDAHLyp ++++ return 0 +++ [[ xsome-name-11182 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ij7PB3FNsn +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.rYyogEvySd ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ij7PB3FNsn ++++ cat /tmp/tmp.rYyogEvySd ++++ rm /tmp/tmp.ij7PB3FNsn /tmp/tmp.rYyogEvySd ++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.u8bNr5jA3J +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2VcqASfvVR ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.u8bNr5jA3J ++++ cat /tmp/tmp.2VcqASfvVR ++++ rm /tmp/tmp.u8bNr5jA3J /tmp/tmp.2VcqASfvVR ++++ return 0 +++ [[ xsome-name-11182-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.mVY4UH2ZlQ ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.aM32WLLodq +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.mVY4UH2ZlQ +++++ cat /tmp/tmp.aM32WLLodq +++++ rm /tmp/tmp.mVY4UH2ZlQ /tmp/tmp.aM32WLLodq +++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.j8NfzF8Lz7 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.7mPFj0fa51 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.j8NfzF8Lz7 ++++ cat /tmp/tmp.7mPFj0fa51 ++++ rm /tmp/tmp.j8NfzF8Lz7 /tmp/tmp.7mPFj0fa51 ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-11182.scheduled-backup-13036 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n1HEZe4bjQ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xj4ZVdfNfO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.n1HEZe4bjQ +++ cat /tmp/tmp.xj4ZVdfNfO +++ rm /tmp/tmp.n1HEZe4bjQ /tmp/tmp.xj4ZVdfNfO +++ return 0 ++ local client_container=pg-client-76df9df876-pthwj ++ kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.tPHVWvg4kS +++ mktemp ++ local LAST_ERR=/tmp/tmp.5xEk2bkC97 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.tPHVWvg4kS ++ cat /tmp/tmp.5xEk2bkC97 ++ rm /tmp/tmp.tPHVWvg4kS /tmp/tmp.5xEk2bkC97 ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.E4UuhjGN4R ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ml4KnRgkAA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.E4UuhjGN4R +++ cat /tmp/tmp.ml4KnRgkAA +++ rm /tmp/tmp.E4UuhjGN4R /tmp/tmp.ml4KnRgkAA +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7ynjz7BGwO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.54suQfGZ5y +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7ynjz7BGwO +++ cat /tmp/tmp.54suQfGZ5y +++ rm /tmp/tmp.7ynjz7BGwO /tmp/tmp.54suQfGZ5y +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.13Z3qMZUKq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4atbT3Evlq +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.13Z3qMZUKq +++ cat /tmp/tmp.4atbT3Evlq +++ rm /tmp/tmp.13Z3qMZUKq /tmp/tmp.4atbT3Evlq +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.GBnfB8Bws5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.zrKuUM3Tek ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.GBnfB8Bws5 ++++ cat /tmp/tmp.zrKuUM3Tek ++++ rm /tmp/tmp.GBnfB8Bws5 /tmp/tmp.zrKuUM3Tek ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RFqU454BbG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RajZ7z22Ga +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RFqU454BbG +++ cat /tmp/tmp.RajZ7z22Ga +++ rm /tmp/tmp.RFqU454BbG /tmp/tmp.RajZ7z22Ga +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.EGkIfuqjVa +++ mktemp ++ local LAST_ERR=/tmp/tmp.soPl2agLWX ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.EGkIfuqjVa ++ cat /tmp/tmp.soPl2agLWX ++ rm /tmp/tmp.EGkIfuqjVa /tmp/tmp.soPl2agLWX ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.JASPJbC1Co ++ mktemp + local LAST_ERR=/tmp/tmp.WTV87jjH0b + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.JASPJbC1Co + cat /tmp/tmp.WTV87jjH0b + rm /tmp/tmp.JASPJbC1Co /tmp/tmp.WTV87jjH0b + return 0 + run_restore some-name-11182 full s3 773 xid + local cluster=some-name-11182 + local rstr_prefix=full + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-11182 + yq w - metadata.name full-some-name-11182 + yq w - spec.name full-some-name-11182 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.parameters.backrest-restore-cluster some-name-11182 + yq w - spec.namespace scheduled-backup-13036 + [[ 773 != \n\u\l\l ]] + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + yq w /tmp/tmp.7cfNRxU4oI/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + mv /tmp/tmp.7cfNRxU4oI/restore.pitr.yml /tmp/tmp.7cfNRxU4oI/restore.yml + kubectl apply -f /tmp/tmp.7cfNRxU4oI/restore.yml pgtask.pg.percona.com/full-some-name-11182 created + wait_bootstrap_completeness some-name-11182 + local cluster=some-name-11182 + wait_job_completion some-name-11182-bootstrap + local job=some-name-11182-bootstrap + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 4 -ge 120 ]] + [[ 4 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' some-name-11182-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-11182-backrest-shared-repo + local name=some-name-11182-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-11182-backrest-shared-repo + wait_deployment some-name-11182 + local name=some-name-11182 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-11182 + wait_job_completion some-name-11182-stanza-create false + local job=some-name-11182-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 4 -ge 120 ]] + [[ 4 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 5 -ge 120 ]] + [[ 5 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 6 -ge 120 ]] + [[ 6 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 7 -ge 120 ]] + [[ 7 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 8 -ge 120 ]] + [[ 8 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 9 -ge 120 ]] + [[ 9 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 10 -ge 120 ]] + [[ 10 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 11 -ge 120 ]] + [[ 11 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 12 -ge 120 ]] + [[ 12 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 13 -ge 120 ]] + [[ 13 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 14 -ge 120 ]] + [[ 14 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 15 -ge 120 ]] + [[ 15 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 16 -ge 120 ]] + [[ 16 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 17 -ge 120 ]] + [[ 17 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 18 -ge 120 ]] + [[ 18 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 19 -ge 120 ]] + [[ 19 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 20 -ge 120 ]] + [[ 20 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 21 -ge 120 ]] + [[ 21 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 22 -ge 120 ]] + [[ 22 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 23 -ge 120 ]] + [[ 23 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 24 -ge 120 ]] + [[ 24 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 25 -ge 120 ]] + [[ 25 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 26 -ge 120 ]] + [[ 26 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 27 -ge 120 ]] + [[ 27 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 28 -ge 120 ]] + [[ 28 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 29 -ge 120 ]] + [[ 29 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 30 -ge 120 ]] + [[ 30 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 31 -ge 120 ]] + [[ 31 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 32 -ge 120 ]] + [[ 32 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 33 -ge 120 ]] + [[ 33 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 34 -ge 120 ]] + [[ 34 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 35 -ge 120 ]] + [[ 35 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 36 -ge 120 ]] + [[ 36 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 37 -ge 120 ]] + [[ 37 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 38 -ge 120 ]] + [[ 38 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 39 -ge 120 ]] + [[ 39 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 40 -ge 120 ]] + [[ 40 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 41 -ge 120 ]] + [[ 41 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 42 -ge 120 ]] + [[ 42 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 43 -ge 120 ]] + [[ 43 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 44 -ge 120 ]] + [[ 44 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 45 -ge 120 ]] + [[ 45 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 46 -ge 120 ]] + [[ 46 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 47 -ge 120 ]] + [[ 47 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 48 -ge 120 ]] + [[ 48 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 49 -ge 120 ]] + [[ 49 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 50 -ge 120 ]] + [[ 50 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 51 -ge 120 ]] + [[ 51 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 52 -ge 120 ]] + [[ 52 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 53 -ge 120 ]] + [[ 53 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 54 -ge 120 ]] + [[ 54 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 55 -ge 120 ]] + [[ 55 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 56 -ge 120 ]] + [[ 56 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 57 -ge 120 ]] + [[ 57 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 58 -ge 120 ]] + [[ 58 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 59 -ge 120 ]] + [[ 59 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 60 -ge 120 ]] + [[ 60 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 61 -ge 120 ]] + [[ 61 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 62 -ge 120 ]] + [[ 62 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 63 -ge 120 ]] + [[ 63 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 64 -ge 120 ]] + [[ 64 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 65 -ge 120 ]] + [[ 65 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 66 -ge 120 ]] + [[ 66 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 67 -ge 120 ]] + [[ 67 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 68 -ge 120 ]] + [[ 68 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 69 -ge 120 ]] + [[ 69 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 70 -ge 120 ]] + [[ 70 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 71 -ge 120 ]] + [[ 71 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 72 -ge 120 ]] + [[ 72 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 73 -ge 120 ]] + [[ 73 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 74 -ge 120 ]] + [[ 74 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 75 -ge 120 ]] + [[ 75 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 76 -ge 120 ]] + [[ 76 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 77 -ge 120 ]] + [[ 77 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 78 -ge 120 ]] + [[ 78 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 79 -ge 120 ]] + [[ 79 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 80 -ge 120 ]] + [[ 80 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 81 -ge 120 ]] + [[ 81 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 82 -ge 120 ]] + [[ 82 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 83 -ge 120 ]] + [[ 83 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 84 -ge 120 ]] + [[ 84 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 85 -ge 120 ]] + [[ 85 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 86 -ge 120 ]] + [[ 86 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 87 -ge 120 ]] + [[ 87 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 88 -ge 120 ]] + [[ 88 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 89 -ge 120 ]] + [[ 89 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 90 -ge 120 ]] + [[ 90 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 91 -ge 120 ]] + [[ 91 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 92 -ge 120 ]] + [[ 92 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 93 -ge 120 ]] + [[ 93 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 94 -ge 120 ]] + [[ 94 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 95 -ge 120 ]] + [[ 95 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 96 -ge 120 ]] + [[ 96 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 97 -ge 120 ]] + [[ 97 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 98 -ge 120 ]] + [[ 98 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 99 -ge 120 ]] + [[ 99 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 100 -ge 120 ]] + [[ 100 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 101 -ge 120 ]] + [[ 101 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 102 -ge 120 ]] + [[ 102 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 103 -ge 120 ]] + [[ 103 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 104 -ge 120 ]] + [[ 104 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 105 -ge 120 ]] + [[ 105 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 106 -ge 120 ]] + [[ 106 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 107 -ge 120 ]] + [[ 107 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 108 -ge 120 ]] + [[ 108 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 109 -ge 120 ]] + [[ 109 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 110 -ge 120 ]] + [[ 110 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 111 -ge 120 ]] + [[ 111 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 112 -ge 120 ]] + [[ 112 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 113 -ge 120 ]] + [[ 113 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 114 -ge 120 ]] + [[ 114 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 115 -ge 120 ]] + [[ 115 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 116 -ge 120 ]] + [[ 116 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 117 -ge 120 ]] + [[ 117 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 118 -ge 120 ]] + [[ 118 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 119 -ge 120 ]] + [[ 119 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-stanza-create -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 120 -ge 120 ]] + [[ false == \t\r\u\e ]] + [[ 120 -ge 120 ]] + [[ false != \t\r\u\e ]] + echo Can not detect job. Passing by. Can not detect job. Passing by. + set -o xtrace + return 0 + wait_job_completion backrest-backup-some-name-11182 + local job=backrest-backup-some-name-11182 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-11182+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-11182 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-11182 ++ local user=postgres ++ local cluster=some-name-11182 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c3dm966UIZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.KFlXM7gAa9 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.c3dm966UIZ +++ cat /tmp/tmp.KFlXM7gAa9 +++ rm /tmp/tmp.c3dm966UIZ /tmp/tmp.KFlXM7gAa9 +++ return 0 ++ [[ xsome-name-11182 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bkFE3EFCos ++++ mktemp +++ local LAST_ERR=/tmp/tmp.43zYxrDecG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-11182 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bkFE3EFCos +++ cat /tmp/tmp.43zYxrDecG +++ rm /tmp/tmp.bkFE3EFCos /tmp/tmp.43zYxrDecG +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XMQKSHWLin ++++ mktemp +++ local LAST_ERR=/tmp/tmp.SM8rAlpfXr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XMQKSHWLin +++ cat /tmp/tmp.SM8rAlpfXr +++ rm /tmp/tmp.XMQKSHWLin /tmp/tmp.SM8rAlpfXr +++ return 0 ++ [[ xsome-name-11182-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gF0oexduaC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Uk2cKRinxC ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-11182 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.gF0oexduaC ++++ cat /tmp/tmp.Uk2cKRinxC ++++ rm /tmp/tmp.gF0oexduaC /tmp/tmp.Uk2cKRinxC ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lfu5BHY7hp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V78dPEGwWb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-11182-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lfu5BHY7hp +++ cat /tmp/tmp.V78dPEGwWb +++ rm /tmp/tmp.lfu5BHY7hp /tmp/tmp.V78dPEGwWb +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-11182.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-11182.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.s0aLq1z6x0 +++ mktemp ++ local LAST_ERR=/tmp/tmp.DH0AhYS6E8 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.s0aLq1z6x0 ++ cat /tmp/tmp.DH0AhYS6E8 ++ rm /tmp/tmp.s0aLq1z6x0 /tmp/tmp.DH0AhYS6E8 ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.Sqe1mcUya6 ++ mktemp + local LAST_ERR=/tmp/tmp.qrREX1SBXM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-11182.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Sqe1mcUya6 + cat /tmp/tmp.qrREX1SBXM + rm /tmp/tmp.Sqe1mcUya6 /tmp/tmp.qrREX1SBXM + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql + delete_cluster_gracefully some-name-11182 + local cluster=some-name-11182 + kubectl_bin delete perconapgcluster/some-name-11182 ++ mktemp + local LAST_OUT=/tmp/tmp.TFe5f9D4nO ++ mktemp + local LAST_ERR=/tmp/tmp.C8bvGcn38V + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete perconapgcluster/some-name-11182 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.TFe5f9D4nO perconapgcluster.pg.percona.com "some-name-11182" deleted + cat /tmp/tmp.C8bvGcn38V + rm /tmp/tmp.TFe5f9D4nO /tmp/tmp.C8bvGcn38V + return 0 ++ kubectl_bin -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-11182,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.eHTweFjJTr +++ mktemp ++ local LAST_ERR=/tmp/tmp.s8LTiemMjX ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-11182,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.eHTweFjJTr ++ cat /tmp/tmp.s8LTiemMjX ++ rm /tmp/tmp.eHTweFjJTr /tmp/tmp.s8LTiemMjX ++ return 0 + rmjob_name=some-name-11182-rmdata-jurn + wait_job_completion some-name-11182-rmdata-jurn false + local job=some-name-11182-rmdata-jurn + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' some-name-11182-rmdata-jurn+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-11182-rmdata-jurn -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing diff backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing diff backup/restore from s3 ----------------------------------------------------------------------------------- + cluster=some-name-17978 + apply_configs some-name-17978 + local cluster=some-name-17978 + set +x secret/some-name-17978-backrest-repo-config created + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/custom-config.yml + yq w - metadata.name some-name-17978-custom-config + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.Tffl4Ub30O ++ mktemp + local LAST_ERR=/tmp/tmp.ypANqzMCZa + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Tffl4Ub30O configmap/some-name-17978-custom-config created + cat /tmp/tmp.ypANqzMCZa + rm /tmp/tmp.Tffl4Ub30O /tmp/tmp.ypANqzMCZa + return 0 + spinup_pgcluster some-name-17978 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml s3 some-name-17978-custom-config false true diff + local cluster=some-name-17978 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-17978-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=diff + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=scheduled-backup-13036 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-17978 scheduled-backup-13036 + local cluster=some-name-17978 + local namespace=scheduled-backup-13036 + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - metadata.name some-name-17978-users-secret + yq w - stringData.some-name some-name_pass + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.y5lK1UrOkM ++ mktemp + local LAST_ERR=/tmp/tmp.lGPVunyu9m + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.y5lK1UrOkM secret/some-name-17978-users-secret created + cat /tmp/tmp.lGPVunyu9m + rm /tmp/tmp.y5lK1UrOkM /tmp/tmp.lGPVunyu9m + return 0 + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - metadata.name some-name-17978-pmm-secret + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.kiznpHRbyx ++ mktemp + local LAST_ERR=/tmp/tmp.L3sa6DykQ6 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.kiznpHRbyx secret/some-name-17978-pmm-secret created + cat /tmp/tmp.L3sa6DykQ6 + rm /tmp/tmp.kiznpHRbyx /tmp/tmp.L3sa6DykQ6 + return 0 + kubectl_bin -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.89ZbJYfE8K ++ mktemp + local LAST_ERR=/tmp/tmp.qYnqhYwfQZ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.89ZbJYfE8K deployment.apps/pg-client unchanged + cat /tmp/tmp.qYnqhYwfQZ + rm /tmp/tmp.89ZbJYfE8K /tmp/tmp.qYnqhYwfQZ + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name-17978 s3 false false some-name-17978-custom-config false true diff false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name-17978 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-17978-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=diff + local tls_only=false + local root_via_pgbouncer= + yq w - metadata.labels.crunchy-pgha-scope some-name-17978 + yq w - metadata.labels.deployment-name some-name-17978 + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name-17978 + yq w - metadata.name some-name-17978 + yq w - spec.clustername some-name-17978 + yq w - spec.database some-name-17978 + yq w - metadata.labels.pg-cluster some-name-17978 + yq w - spec.name some-name-17978 + yq w - spec.user some-name + yq w - spec.keepData false + yq w - spec.keepBackups false + yq w - metadata.labels.name some-name-17978 + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.secretsName some-name-17978-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + case ${podAntiAffinity} in + case ${backup} in + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + [[ true == \t\r\u\e ]] + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.schedule[0].type' diff + mv /tmp/tmp.7cfNRxU4oI/cr.backup.yaml /tmp/tmp.7cfNRxU4oI/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-17978-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.7cfNRxU4oI/cr.yaml spec.pgPrimary.customconfig some-name-17978-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.KvbHWxfzbE ++ mktemp + local LAST_ERR=/tmp/tmp.YchOvcpJoU + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.KvbHWxfzbE perconapgcluster.pg.percona.com/some-name-17978 created + cat /tmp/tmp.YchOvcpJoU + rm /tmp/tmp.KvbHWxfzbE /tmp/tmp.YchOvcpJoU + return 0 + wait_deployment some-name-17978-backrest-shared-repo + local name=some-name-17978-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-17978-backrest-shared-repo... + wait_deployment some-name-17978 + local name=some-name-17978 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-17978..... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-17978-stanza-create false + local job=some-name-17978-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' some-name-17978-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-17978 + local job=backrest-backup-some-name-17978 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-17978+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-17978 + cluster_name=some-name-17978 + wait_cluster_status some-name-17978 'pgcluster Initialized' + local cluster_name=some-name-17978 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-13036 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.NL1SbetRlt +++ mktemp ++ local LAST_ERR=/tmp/tmp.B4PwUU4SIH ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.NL1SbetRlt ++ cat /tmp/tmp.B4PwUU4SIH ++ rm /tmp/tmp.NL1SbetRlt /tmp/tmp.B4PwUU4SIH ++ return 0 + status_query='kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Msca2gaLT4 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.abHPLrH6jx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Msca2gaLT4 +++ cat /tmp/tmp.abHPLrH6jx +++ rm /tmp/tmp.Msca2gaLT4 /tmp/tmp.abHPLrH6jx +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NgylJllhbM ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VInGZoILGV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NgylJllhbM +++ cat /tmp/tmp.VInGZoILGV +++ rm /tmp/tmp.NgylJllhbM /tmp/tmp.VInGZoILGV +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.SPVd54ZA4m ++++ mktemp +++ local LAST_ERR=/tmp/tmp.FqyzkErLtj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.SPVd54ZA4m +++ cat /tmp/tmp.FqyzkErLtj +++ rm /tmp/tmp.SPVd54ZA4m /tmp/tmp.FqyzkErLtj +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.tDFhsJL3IA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.stfeSywfee +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.tDFhsJL3IA +++ cat /tmp/tmp.stfeSywfee +++ rm /tmp/tmp.tDFhsJL3IA /tmp/tmp.stfeSywfee +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.8iThT7jQga +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.hwrt5D674t ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.8iThT7jQga ++++ cat /tmp/tmp.hwrt5D674t ++++ rm /tmp/tmp.8iThT7jQga /tmp/tmp.hwrt5D674t ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QIQB9Y5ouL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zPMXCzBhJk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.QIQB9Y5ouL +++ cat /tmp/tmp.zPMXCzBhJk +++ rm /tmp/tmp.QIQB9Y5ouL /tmp/tmp.zPMXCzBhJk +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.1QK4kkldYa +++ mktemp ++ local LAST_ERR=/tmp/tmp.FFxENtVaHm ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.1QK4kkldYa ++ cat /tmp/tmp.FFxENtVaHm ++ rm /tmp/tmp.1QK4kkldYa /tmp/tmp.FFxENtVaHm ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.2LEmzsih0f ++ mktemp + local LAST_ERR=/tmp/tmp.6c6FANkyDu + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.2LEmzsih0f + cat /tmp/tmp.6c6FANkyDu + rm /tmp/tmp.2LEmzsih0f /tmp/tmp.6c6FANkyDu + return 0 ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.j2npJILvFc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ypTPEqAdwB +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.j2npJILvFc +++ cat /tmp/tmp.ypTPEqAdwB +++ rm /tmp/tmp.j2npJILvFc /tmp/tmp.ypTPEqAdwB +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.blGTW8cL0N ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZM3wMXfslo +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.blGTW8cL0N +++ cat /tmp/tmp.ZM3wMXfslo +++ rm /tmp/tmp.blGTW8cL0N /tmp/tmp.ZM3wMXfslo +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ScEGnI97P2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gHZHd8XTen +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ScEGnI97P2 +++ cat /tmp/tmp.gHZHd8XTen +++ rm /tmp/tmp.ScEGnI97P2 /tmp/tmp.gHZHd8XTen +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ovQ3bYUpcB +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.RUPRmZWxGa ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ovQ3bYUpcB ++++ cat /tmp/tmp.RUPRmZWxGa ++++ rm /tmp/tmp.ovQ3bYUpcB /tmp/tmp.RUPRmZWxGa ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xrPNVRbilX ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IECE12xZoQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.xrPNVRbilX +++ cat /tmp/tmp.IECE12xZoQ +++ rm /tmp/tmp.xrPNVRbilX /tmp/tmp.IECE12xZoQ +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.oRqPO4k2pC +++ mktemp ++ local LAST_ERR=/tmp/tmp.OakbUZYBKt ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.oRqPO4k2pC ++ cat /tmp/tmp.OakbUZYBKt ++ rm /tmp/tmp.oRqPO4k2pC /tmp/tmp.OakbUZYBKt ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.TiKqkjV3Ck ++ mktemp + local LAST_ERR=/tmp/tmp.6q4MyoAooH + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.TiKqkjV3Ck + cat /tmp/tmp.6q4MyoAooH + rm /tmp/tmp.TiKqkjV3Ck /tmp/tmp.6q4MyoAooH + return 0 ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7mMaCVYEck ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vZq0Vj5Qyh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7mMaCVYEck +++ cat /tmp/tmp.vZq0Vj5Qyh +++ rm /tmp/tmp.7mMaCVYEck /tmp/tmp.vZq0Vj5Qyh +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ICrj2XW3oO ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1ztCLO6GQI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ICrj2XW3oO +++ cat /tmp/tmp.1ztCLO6GQI +++ rm /tmp/tmp.ICrj2XW3oO /tmp/tmp.1ztCLO6GQI +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.QKPQpjWnTI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.efE3y3CNYZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.QKPQpjWnTI +++ cat /tmp/tmp.efE3y3CNYZ +++ rm /tmp/tmp.QKPQpjWnTI /tmp/tmp.efE3y3CNYZ +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.EnAHUZxhjr +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IRhuO7XdUe ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.EnAHUZxhjr ++++ cat /tmp/tmp.IRhuO7XdUe ++++ rm /tmp/tmp.EnAHUZxhjr /tmp/tmp.IRhuO7XdUe ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Ptp5twaVCA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.29bTldUTIw +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Ptp5twaVCA +++ cat /tmp/tmp.29bTldUTIw +++ rm /tmp/tmp.Ptp5twaVCA /tmp/tmp.29bTldUTIw +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mdxDLbaYkC +++ mktemp ++ local LAST_ERR=/tmp/tmp.D8r5x2z7SQ ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.mdxDLbaYkC ++ cat /tmp/tmp.D8r5x2z7SQ ++ rm /tmp/tmp.mdxDLbaYkC /tmp/tmp.D8r5x2z7SQ ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.pGUPW8z28F ++ mktemp + local LAST_ERR=/tmp/tmp.W1acDNenf1 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.pGUPW8z28F + cat /tmp/tmp.W1acDNenf1 + rm /tmp/tmp.pGUPW8z28F /tmp/tmp.W1acDNenf1 + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bLgbL0CpO6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8ibAZLQptf +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bLgbL0CpO6 +++ cat /tmp/tmp.8ibAZLQptf +++ rm /tmp/tmp.bLgbL0CpO6 /tmp/tmp.8ibAZLQptf +++ return 0 ++ next_backup_after_sec some-name-17978-sch-backup ++ local conf_map_name=some-name-17978-sch-backup ++++ kubectl_bin get configmap/some-name-17978-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.1gY31Baw9j +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.HcJAXP4K6N ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get configmap/some-name-17978-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.1gY31Baw9j ++++ cat /tmp/tmp.HcJAXP4K6N ++++ rm /tmp/tmp.1gY31Baw9j /tmp/tmp.HcJAXP4K6N ++++ return 0 +++ /usr/bin/date +%s --date=2024-05-24T17:54:52Z --utc ++ local configmap_creation_time=1716573292 +++ kubectl_bin get configmap/some-name-17978-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.zv6WjxUdbp ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sm3deBYDPD +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get configmap/some-name-17978-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.zv6WjxUdbp +++ cat /tmp/tmp.sm3deBYDPD +++ rm /tmp/tmp.zv6WjxUdbp /tmp/tmp.sm3deBYDPD +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1716573292 300 +++ local start_time_sec=1716573292 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716573592 +++ local next_interval_min_edge=292 +++ local next_backup_at_sec=1716573300 +++ echo 1716573300 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 17:58:29 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 17:58:29 GMT ' --utc +%s +++ echo 1716573509 ++ local result=-209 ++ [[ -209 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Fri, 24 May 2024 17:58:30 GMT ' +++++ /usr/bin/date --date 'Fri, 24 May 2024 17:58:30 GMT ' --utc +%s ++++ echo 1716573510 +++ calculate_next_backup_start 1716573510 300 +++ local start_time_sec=1716573510 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716573810 +++ local next_interval_min_edge=210 +++ local next_backup_at_sec=1716573600 +++ echo 1716573600 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 17:58:30 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 17:58:30 GMT ' --utc +%s +++ echo 1716573510 ++ result=90 ++ [[ 90 -gt 0 ]] ++ echo 90 + sleep 90 + kubectl_bin wait job/some-name-17978-diff-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.IPDSC7KsU3 ++ mktemp + local LAST_ERR=/tmp/tmp.aeRV3p755k + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl wait job/some-name-17978-diff-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.IPDSC7KsU3 job.batch/some-name-17978-diff-sch-backup condition met + cat /tmp/tmp.aeRV3p755k + rm /tmp/tmp.IPDSC7KsU3 /tmp/tmp.aeRV3p755k + return 0 + restore some-name-17978 s3 diff + local cluster=some-name-17978 + local storage=s3 + local backup_type=diff + local restore_target= ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2OLt8RfG0I ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZxMVGGtjnb +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2OLt8RfG0I +++ cat /tmp/tmp.ZxMVGGtjnb +++ rm /tmp/tmp.2OLt8RfG0I /tmp/tmp.ZxMVGGtjnb +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qWEt91hAWS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DinNBIiTpl +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.qWEt91hAWS +++ cat /tmp/tmp.DinNBIiTpl +++ rm /tmp/tmp.qWEt91hAWS /tmp/tmp.DinNBIiTpl +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.xSH4AerxUA ++++ mktemp +++ local LAST_ERR=/tmp/tmp.A4eslxKCUL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.xSH4AerxUA +++ cat /tmp/tmp.A4eslxKCUL +++ rm /tmp/tmp.xSH4AerxUA /tmp/tmp.A4eslxKCUL +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.sMPLo99kXP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.v6FvtsXPVS ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.sMPLo99kXP ++++ cat /tmp/tmp.v6FvtsXPVS ++++ rm /tmp/tmp.sMPLo99kXP /tmp/tmp.v6FvtsXPVS ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FFO382P8nY ++++ mktemp +++ local LAST_ERR=/tmp/tmp.G8suomRKXP +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.FFO382P8nY +++ cat /tmp/tmp.G8suomRKXP +++ rm /tmp/tmp.FFO382P8nY /tmp/tmp.G8suomRKXP +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.poQNDDzUDn +++ mktemp ++ local LAST_ERR=/tmp/tmp.dk9fuG4D1s ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.poQNDDzUDn ++ cat /tmp/tmp.dk9fuG4D1s ++ rm /tmp/tmp.poQNDDzUDn /tmp/tmp.dk9fuG4D1s ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.7c9q8sbqZH ++ mktemp + local LAST_ERR=/tmp/tmp.qRG7uCNePi + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7c9q8sbqZH + cat /tmp/tmp.qRG7uCNePi + rm /tmp/tmp.7c9q8sbqZH /tmp/tmp.qRG7uCNePi + return 0 ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.B3hqB3IzV8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.GvJNdkOaH9 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.B3hqB3IzV8 +++ cat /tmp/tmp.GvJNdkOaH9 +++ rm /tmp/tmp.B3hqB3IzV8 /tmp/tmp.GvJNdkOaH9 +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.91GJEbr5Hn ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zrbgOzBTdR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.91GJEbr5Hn +++ cat /tmp/tmp.zrbgOzBTdR +++ rm /tmp/tmp.91GJEbr5Hn /tmp/tmp.zrbgOzBTdR +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.b55Ctiytle ++++ mktemp +++ local LAST_ERR=/tmp/tmp.4znIsyVEdL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.b55Ctiytle +++ cat /tmp/tmp.4znIsyVEdL +++ rm /tmp/tmp.b55Ctiytle /tmp/tmp.4znIsyVEdL +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.zsW25460oG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.DYvsI2CFYc ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.zsW25460oG ++++ cat /tmp/tmp.DYvsI2CFYc ++++ rm /tmp/tmp.zsW25460oG /tmp/tmp.DYvsI2CFYc ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uKbhbgevuG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qxAunlxchZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uKbhbgevuG +++ cat /tmp/tmp.qxAunlxchZ +++ rm /tmp/tmp.uKbhbgevuG /tmp/tmp.qxAunlxchZ +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-17978.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.3v4EArprdI +++ mktemp ++ local LAST_ERR=/tmp/tmp.XlKusgsfag ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.3v4EArprdI ++ cat /tmp/tmp.XlKusgsfag ++ rm /tmp/tmp.3v4EArprdI /tmp/tmp.XlKusgsfag ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.jVjDKO38OT ++ mktemp + local LAST_ERR=/tmp/tmp.fq85rOdP8r + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jVjDKO38OT + cat /tmp/tmp.fq85rOdP8r + rm /tmp/tmp.jVjDKO38OT /tmp/tmp.fq85rOdP8r + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.7kgbMCYHY0 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.NfJQZwsDNV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.7kgbMCYHY0 +++ cat /tmp/tmp.NfJQZwsDNV +++ rm /tmp/tmp.7kgbMCYHY0 /tmp/tmp.NfJQZwsDNV +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.12ZXfbJ8jd ++++ mktemp +++ local LAST_ERR=/tmp/tmp.yhljaHx7t6 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.12ZXfbJ8jd +++ cat /tmp/tmp.yhljaHx7t6 +++ rm /tmp/tmp.12ZXfbJ8jd /tmp/tmp.yhljaHx7t6 +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.laquW2ejca ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2gClBHaSbr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.laquW2ejca +++ cat /tmp/tmp.2gClBHaSbr +++ rm /tmp/tmp.laquW2ejca /tmp/tmp.2gClBHaSbr +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.upZip8C6Za +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.nipT6i8kLt ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.upZip8C6Za ++++ cat /tmp/tmp.nipT6i8kLt ++++ rm /tmp/tmp.upZip8C6Za /tmp/tmp.nipT6i8kLt ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ylFzGfQ5BP ++++ mktemp +++ local LAST_ERR=/tmp/tmp.xlE5tDUoO5 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ylFzGfQ5BP +++ cat /tmp/tmp.xlE5tDUoO5 +++ rm /tmp/tmp.ylFzGfQ5BP /tmp/tmp.xlE5tDUoO5 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.L5SIZUoV8w +++ mktemp ++ local LAST_ERR=/tmp/tmp.jTpFKcMnNI ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.L5SIZUoV8w ++ cat /tmp/tmp.jTpFKcMnNI ++ rm /tmp/tmp.L5SIZUoV8w /tmp/tmp.jTpFKcMnNI ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.UDLVtDWEyF ++ mktemp + local LAST_ERR=/tmp/tmp.Cm7VIyHIpO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.UDLVtDWEyF + cat /tmp/tmp.Cm7VIyHIpO + rm /tmp/tmp.UDLVtDWEyF /tmp/tmp.Cm7VIyHIpO + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-17978 +++ local user=postgres +++ local cluster=some-name-17978 +++ local namespace=scheduled-backup-13036 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.0Es4ow3YvT +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IT5CSZF9zN ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.0Es4ow3YvT ++++ cat /tmp/tmp.IT5CSZF9zN ++++ rm /tmp/tmp.0Es4ow3YvT /tmp/tmp.IT5CSZF9zN ++++ return 0 +++ [[ xsome-name-17978 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.49i3pU6JF5 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.LacQo6tI84 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.49i3pU6JF5 ++++ cat /tmp/tmp.LacQo6tI84 ++++ rm /tmp/tmp.49i3pU6JF5 /tmp/tmp.LacQo6tI84 ++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.P1fpX8v5Lu +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.rynBkwp6bz ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.P1fpX8v5Lu ++++ cat /tmp/tmp.rynBkwp6bz ++++ rm /tmp/tmp.P1fpX8v5Lu /tmp/tmp.rynBkwp6bz ++++ return 0 +++ [[ xsome-name-17978-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.euUfRVUyvn ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.s4pbaqK9pf +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.euUfRVUyvn +++++ cat /tmp/tmp.s4pbaqK9pf +++++ rm /tmp/tmp.euUfRVUyvn /tmp/tmp.s4pbaqK9pf +++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.HMmKsqX7Y0 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.2J2jillmy1 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.HMmKsqX7Y0 ++++ cat /tmp/tmp.2J2jillmy1 ++++ rm /tmp/tmp.HMmKsqX7Y0 /tmp/tmp.2J2jillmy1 ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-17978.scheduled-backup-13036 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Aa5ue4BPvC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wxxY6SWk1D +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Aa5ue4BPvC +++ cat /tmp/tmp.wxxY6SWk1D +++ rm /tmp/tmp.Aa5ue4BPvC /tmp/tmp.wxxY6SWk1D +++ return 0 ++ local client_container=pg-client-76df9df876-pthwj ++ kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.dOt2PZNzOs +++ mktemp ++ local LAST_ERR=/tmp/tmp.Hntb77TO9T ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.dOt2PZNzOs ++ cat /tmp/tmp.Hntb77TO9T ++ rm /tmp/tmp.dOt2PZNzOs /tmp/tmp.Hntb77TO9T ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.oFhQWJlk2h ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L4puCI5NjH +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.oFhQWJlk2h +++ cat /tmp/tmp.L4puCI5NjH +++ rm /tmp/tmp.oFhQWJlk2h /tmp/tmp.L4puCI5NjH +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.h8AX2gwWdF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ysBM3SNpb3 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.h8AX2gwWdF +++ cat /tmp/tmp.ysBM3SNpb3 +++ rm /tmp/tmp.h8AX2gwWdF /tmp/tmp.ysBM3SNpb3 +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.PlxUaBgf1K ++++ mktemp +++ local LAST_ERR=/tmp/tmp.TVyDorz2xO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.PlxUaBgf1K +++ cat /tmp/tmp.TVyDorz2xO +++ rm /tmp/tmp.PlxUaBgf1K /tmp/tmp.TVyDorz2xO +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.BKsHQtgnbt +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.GrNfZrK1fX ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.BKsHQtgnbt ++++ cat /tmp/tmp.GrNfZrK1fX ++++ rm /tmp/tmp.BKsHQtgnbt /tmp/tmp.GrNfZrK1fX ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ctbHpO8ea2 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DGmBRWA5tr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ctbHpO8ea2 +++ cat /tmp/tmp.DGmBRWA5tr +++ rm /tmp/tmp.ctbHpO8ea2 /tmp/tmp.DGmBRWA5tr +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.hEdjWTtesJ +++ mktemp ++ local LAST_ERR=/tmp/tmp.e8aMYjHJfk ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.hEdjWTtesJ ++ cat /tmp/tmp.e8aMYjHJfk ++ rm /tmp/tmp.hEdjWTtesJ /tmp/tmp.e8aMYjHJfk ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.x7P76Egcog ++ mktemp + local LAST_ERR=/tmp/tmp.Q4NWUR0VsG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.x7P76Egcog + cat /tmp/tmp.Q4NWUR0VsG + rm /tmp/tmp.x7P76Egcog /tmp/tmp.Q4NWUR0VsG + return 0 + run_restore some-name-17978 diff s3 773 xid + local cluster=some-name-17978 + local rstr_prefix=diff + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-17978 + yq w - metadata.name diff-some-name-17978 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.parameters.backrest-restore-cluster some-name-17978 + yq w - spec.namespace scheduled-backup-13036 + yq w - spec.name diff-some-name-17978 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.7cfNRxU4oI/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.7cfNRxU4oI/restore.pitr.yml /tmp/tmp.7cfNRxU4oI/restore.yml + kubectl apply -f /tmp/tmp.7cfNRxU4oI/restore.yml pgtask.pg.percona.com/diff-some-name-17978 created + wait_bootstrap_completeness some-name-17978 + local cluster=some-name-17978 + wait_job_completion some-name-17978-bootstrap + local job=some-name-17978-bootstrap + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 4 -ge 120 ]] + [[ 4 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' some-name-17978-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-bootstrap -o 'jsonpath={.metadata.name}' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-17978-backrest-shared-repo + local name=some-name-17978-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-17978-backrest-shared-repo + wait_deployment some-name-17978 + local name=some-name-17978 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-17978. + wait_job_completion some-name-17978-stanza-create false + local job=some-name-17978-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' some-name-17978-stanza-create+ retry=0 ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-17978 + local job=backrest-backup-some-name-17978 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-17978+ retry=0 ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-17978 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-17978 ++ local user=postgres ++ local cluster=some-name-17978 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.LJCagMh1nV ++++ mktemp +++ local LAST_ERR=/tmp/tmp.elkUxNrWSS +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.LJCagMh1nV +++ cat /tmp/tmp.elkUxNrWSS +++ rm /tmp/tmp.LJCagMh1nV /tmp/tmp.elkUxNrWSS +++ return 0 ++ [[ xsome-name-17978 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lTlBndhqsE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.1QUI5vwlHO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-17978 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lTlBndhqsE +++ cat /tmp/tmp.1QUI5vwlHO +++ rm /tmp/tmp.lTlBndhqsE /tmp/tmp.1QUI5vwlHO +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mNKCrIzqtG ++++ mktemp +++ local LAST_ERR=/tmp/tmp.M9BEfybL6R +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mNKCrIzqtG +++ cat /tmp/tmp.M9BEfybL6R +++ rm /tmp/tmp.mNKCrIzqtG /tmp/tmp.M9BEfybL6R +++ return 0 ++ [[ xsome-name-17978-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.AMHjbRpqkp +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.wD9sULRGGy ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-17978 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.AMHjbRpqkp ++++ cat /tmp/tmp.wD9sULRGGy ++++ rm /tmp/tmp.AMHjbRpqkp /tmp/tmp.wD9sULRGGy ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.VqJhVJtEuq ++++ mktemp +++ local LAST_ERR=/tmp/tmp.i94QwpQxWI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-17978-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.VqJhVJtEuq +++ cat /tmp/tmp.i94QwpQxWI +++ rm /tmp/tmp.VqJhVJtEuq /tmp/tmp.i94QwpQxWI +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-17978.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-17978.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ttDIjfBmQh +++ mktemp ++ local LAST_ERR=/tmp/tmp.4Tgn47sLwK ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ttDIjfBmQh ++ cat /tmp/tmp.4Tgn47sLwK ++ rm /tmp/tmp.ttDIjfBmQh /tmp/tmp.4Tgn47sLwK ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.f3UqmUYgNb ++ mktemp + local LAST_ERR=/tmp/tmp.6Xu2OVYoNP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-17978.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.f3UqmUYgNb + cat /tmp/tmp.6Xu2OVYoNP + rm /tmp/tmp.f3UqmUYgNb /tmp/tmp.6Xu2OVYoNP + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql + delete_cluster_gracefully some-name-17978 + local cluster=some-name-17978 + kubectl_bin delete perconapgcluster/some-name-17978 ++ mktemp + local LAST_OUT=/tmp/tmp.Sdv0arkCZ8 ++ mktemp + local LAST_ERR=/tmp/tmp.zAEKMnmOTc + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete perconapgcluster/some-name-17978 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Sdv0arkCZ8 perconapgcluster.pg.percona.com "some-name-17978" deleted + cat /tmp/tmp.zAEKMnmOTc + rm /tmp/tmp.Sdv0arkCZ8 /tmp/tmp.zAEKMnmOTc + return 0 ++ kubectl_bin -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-17978,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.90yyPmKE6i +++ mktemp ++ local LAST_ERR=/tmp/tmp.MVNIcoplQs ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-17978,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.90yyPmKE6i ++ cat /tmp/tmp.MVNIcoplQs ++ rm /tmp/tmp.90yyPmKE6i /tmp/tmp.MVNIcoplQs ++ return 0 + rmjob_name=some-name-17978-rmdata-kkbf + wait_job_completion some-name-17978-rmdata-kkbf false + local job=some-name-17978-rmdata-kkbf + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' some-name-17978-rmdata-kkbf+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-17978-rmdata-kkbf -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing incr backup/restore from s3' + set +o xtrace ----------------------------------------------------------------------------------- Testing incr backup/restore from s3 ----------------------------------------------------------------------------------- + cluster=some-name-16172 + apply_configs some-name-16172 + local cluster=some-name-16172 + set +x secret/some-name-16172-backrest-repo-config created + yq w - metadata.name some-name-16172-custom-config + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/custom-config.yml + kubectl_bin apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.wNsm3plAqQ ++ mktemp + local LAST_ERR=/tmp/tmp.nrfdAStLIK + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.wNsm3plAqQ configmap/some-name-16172-custom-config created + cat /tmp/tmp.nrfdAStLIK + rm /tmp/tmp.wNsm3plAqQ /tmp/tmp.nrfdAStLIK + return 0 + spinup_pgcluster some-name-16172 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml s3 some-name-16172-custom-config false true incr + local cluster=some-name-16172 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local backup=s3 + local custom_config=some-name-16172-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=incr + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=scheduled-backup-13036 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-16172 scheduled-backup-13036 + local cluster=some-name-16172 + local namespace=scheduled-backup-13036 + yq w - metadata.name some-name-16172-users-secret + kubectl_bin -n scheduled-backup-13036 apply -f - + yq w - stringData.some-name some-name_pass + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.6oGIgcBYEo ++ mktemp + local LAST_ERR=/tmp/tmp.yBVgunFZRE + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.6oGIgcBYEo secret/some-name-16172-users-secret created + cat /tmp/tmp.yBVgunFZRE + rm /tmp/tmp.6oGIgcBYEo /tmp/tmp.yBVgunFZRE + return 0 + yq w - metadata.name some-name-16172-pmm-secret + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.xBTuopMNrT ++ mktemp + local LAST_ERR=/tmp/tmp.te8w5v6Q3t + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.xBTuopMNrT secret/some-name-16172-pmm-secret created + cat /tmp/tmp.te8w5v6Q3t + rm /tmp/tmp.xBTuopMNrT /tmp/tmp.te8w5v6Q3t + return 0 + kubectl_bin -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.1EKzLEeLnV ++ mktemp + local LAST_ERR=/tmp/tmp.0c9urnyh5L + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.1EKzLEeLnV deployment.apps/pg-client unchanged + cat /tmp/tmp.0c9urnyh5L + rm /tmp/tmp.1EKzLEeLnV /tmp/tmp.0c9urnyh5L + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name-16172 s3 false false some-name-16172-custom-config false true incr false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name-16172 + local backup=s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-16172-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=incr + local tls_only=false + local root_via_pgbouncer= + yq w - metadata.labels.crunchy-pgha-scope some-name-16172 + yq w - spec.keepBackups false + yq w - spec.user some-name + yq w - spec.database some-name-16172 + yq w - spec.clustername some-name-16172 + yq w - spec.secretsName some-name-16172-users-secret + yq w - metadata.name some-name-16172 + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name-16172 + yq w - spec.name some-name-16172 + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.keepData false + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - metadata.labels.name some-name-16172 + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - metadata.labels.deployment-name some-name-16172 + yq w - metadata.labels.pg-cluster some-name-16172 + case ${podAntiAffinity} in + case ${backup} in + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].type' s3 + [[ true == \t\r\u\e ]] + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.schedule[0].type' incr + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + mv /tmp/tmp.7cfNRxU4oI/cr.backup.yaml /tmp/tmp.7cfNRxU4oI/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-16172-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.7cfNRxU4oI/cr.yaml spec.pgPrimary.customconfig some-name-16172-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.08q8vbFTmB ++ mktemp + local LAST_ERR=/tmp/tmp.7qRw7Ps2aJ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.08q8vbFTmB perconapgcluster.pg.percona.com/some-name-16172 created + cat /tmp/tmp.7qRw7Ps2aJ + rm /tmp/tmp.08q8vbFTmB /tmp/tmp.7qRw7Ps2aJ + return 0 + wait_deployment some-name-16172-backrest-shared-repo + local name=some-name-16172-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-16172-backrest-shared-repo.. + wait_deployment some-name-16172 + local name=some-name-16172 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-16172..... + [[ s3 == \g\c\s\+* ]] + wait_job_completion some-name-16172-stanza-create false + local job=some-name-16172-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' some-name-16172-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-16172 + local job=backrest-backup-some-name-16172 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-16172+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-16172 + cluster_name=some-name-16172 + wait_cluster_status some-name-16172 'pgcluster Initialized' + local cluster_name=some-name-16172 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-13036 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.mjZZL5e3lB +++ mktemp ++ local LAST_ERR=/tmp/tmp.QdfKefcvgB ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.mjZZL5e3lB ++ cat /tmp/tmp.QdfKefcvgB ++ rm /tmp/tmp.mjZZL5e3lB /tmp/tmp.QdfKefcvgB ++ return 0 + status_query='kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TGZgPq1aiv ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lhfadwVGwk +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TGZgPq1aiv +++ cat /tmp/tmp.lhfadwVGwk +++ rm /tmp/tmp.TGZgPq1aiv /tmp/tmp.lhfadwVGwk +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lAfunkbWZE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.K6nhQ17u5T +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lAfunkbWZE +++ cat /tmp/tmp.K6nhQ17u5T +++ rm /tmp/tmp.lAfunkbWZE /tmp/tmp.K6nhQ17u5T +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.uz872KYtFH ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2CKldX3ciR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.uz872KYtFH +++ cat /tmp/tmp.2CKldX3ciR +++ rm /tmp/tmp.uz872KYtFH /tmp/tmp.2CKldX3ciR +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5kNjlQ5l8k ++++ mktemp +++ local LAST_ERR=/tmp/tmp.z0VeED8Djy +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5kNjlQ5l8k +++ cat /tmp/tmp.z0VeED8Djy +++ rm /tmp/tmp.5kNjlQ5l8k /tmp/tmp.z0VeED8Djy +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.TPIorLuFbg +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.XIzvrjtCug ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.TPIorLuFbg ++++ cat /tmp/tmp.XIzvrjtCug ++++ rm /tmp/tmp.TPIorLuFbg /tmp/tmp.XIzvrjtCug ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.olsTKp8Gg6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.pqWajyylME +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.olsTKp8Gg6 +++ cat /tmp/tmp.pqWajyylME +++ rm /tmp/tmp.olsTKp8Gg6 /tmp/tmp.pqWajyylME +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.sEvBr84kqd +++ mktemp ++ local LAST_ERR=/tmp/tmp.IRuyNxWRs6 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.sEvBr84kqd ++ cat /tmp/tmp.IRuyNxWRs6 ++ rm /tmp/tmp.sEvBr84kqd /tmp/tmp.IRuyNxWRs6 ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.DH8edvC6qa ++ mktemp + local LAST_ERR=/tmp/tmp.JUbVX96DdO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.DH8edvC6qa + cat /tmp/tmp.JUbVX96DdO + rm /tmp/tmp.DH8edvC6qa /tmp/tmp.JUbVX96DdO + return 0 ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.dYp7iMjsma ++++ mktemp +++ local LAST_ERR=/tmp/tmp.knWoB1Sh5j +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.dYp7iMjsma +++ cat /tmp/tmp.knWoB1Sh5j +++ rm /tmp/tmp.dYp7iMjsma /tmp/tmp.knWoB1Sh5j +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CU1198vQv1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MF5vjdr1Z4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CU1198vQv1 +++ cat /tmp/tmp.MF5vjdr1Z4 +++ rm /tmp/tmp.CU1198vQv1 /tmp/tmp.MF5vjdr1Z4 +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.trgWieMhJl ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ckyOkTd0Jh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.trgWieMhJl +++ cat /tmp/tmp.ckyOkTd0Jh +++ rm /tmp/tmp.trgWieMhJl /tmp/tmp.ckyOkTd0Jh +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.umPzs2EK15 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.XfCXcnmNIm ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.umPzs2EK15 ++++ cat /tmp/tmp.XfCXcnmNIm ++++ rm /tmp/tmp.umPzs2EK15 /tmp/tmp.XfCXcnmNIm ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.5Rz8PuNjcI ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oZsS4BKfss +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.5Rz8PuNjcI +++ cat /tmp/tmp.oZsS4BKfss +++ rm /tmp/tmp.5Rz8PuNjcI /tmp/tmp.oZsS4BKfss +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.IS6TJsbSSE +++ mktemp ++ local LAST_ERR=/tmp/tmp.FQSN4AmaDx ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.IS6TJsbSSE ++ cat /tmp/tmp.FQSN4AmaDx ++ rm /tmp/tmp.IS6TJsbSSE /tmp/tmp.FQSN4AmaDx ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.TL3VL6MM9b ++ mktemp + local LAST_ERR=/tmp/tmp.tNmXGuhBpL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.TL3VL6MM9b + cat /tmp/tmp.tNmXGuhBpL + rm /tmp/tmp.TL3VL6MM9b /tmp/tmp.tNmXGuhBpL + return 0 ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DikeIwdrOE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.EQNicQSwGD +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DikeIwdrOE +++ cat /tmp/tmp.EQNicQSwGD +++ rm /tmp/tmp.DikeIwdrOE /tmp/tmp.EQNicQSwGD +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ffYUsuQaHf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bX5YnnTgXj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ffYUsuQaHf +++ cat /tmp/tmp.bX5YnnTgXj +++ rm /tmp/tmp.ffYUsuQaHf /tmp/tmp.bX5YnnTgXj +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Iulgre2L6k ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YC4nI7iIax +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Iulgre2L6k +++ cat /tmp/tmp.YC4nI7iIax +++ rm /tmp/tmp.Iulgre2L6k /tmp/tmp.YC4nI7iIax +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.qcNnsFHYRV +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.7In3n5FEN5 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.qcNnsFHYRV ++++ cat /tmp/tmp.7In3n5FEN5 ++++ rm /tmp/tmp.qcNnsFHYRV /tmp/tmp.7In3n5FEN5 ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.F7vssg2XzC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.zpbPqDwIXO +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.F7vssg2XzC +++ cat /tmp/tmp.zpbPqDwIXO +++ rm /tmp/tmp.F7vssg2XzC /tmp/tmp.zpbPqDwIXO +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.c1njdOojFW +++ mktemp ++ local LAST_ERR=/tmp/tmp.vzKmj8q2bI ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.c1njdOojFW ++ cat /tmp/tmp.vzKmj8q2bI ++ rm /tmp/tmp.c1njdOojFW /tmp/tmp.vzKmj8q2bI ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.NAFqh6lwm5 ++ mktemp + local LAST_ERR=/tmp/tmp.5qE95HZGZc + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.NAFqh6lwm5 + cat /tmp/tmp.5qE95HZGZc + rm /tmp/tmp.NAFqh6lwm5 /tmp/tmp.5qE95HZGZc + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3T8qeh3VHU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wlt4TvSC1b +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.3T8qeh3VHU +++ cat /tmp/tmp.wlt4TvSC1b +++ rm /tmp/tmp.3T8qeh3VHU /tmp/tmp.wlt4TvSC1b +++ return 0 ++ next_backup_after_sec some-name-16172-sch-backup ++ local conf_map_name=some-name-16172-sch-backup ++++ kubectl_bin get configmap/some-name-16172-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.enVFc8VlcW +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.PWGf18ANeY ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get configmap/some-name-16172-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.enVFc8VlcW ++++ cat /tmp/tmp.PWGf18ANeY ++++ rm /tmp/tmp.enVFc8VlcW /tmp/tmp.PWGf18ANeY ++++ return 0 +++ /usr/bin/date +%s --date=2024-05-24T18:06:43Z --utc ++ local configmap_creation_time=1716574003 +++ kubectl_bin get configmap/some-name-16172-sch-backup -o 'jsonpath={.data.schedule}' +++ grep -Eo '[0-9]+' +++ jq .schedule ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DWAhJcTZOt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.8cYesWfJ8a +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get configmap/some-name-16172-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DWAhJcTZOt +++ cat /tmp/tmp.8cYesWfJ8a +++ rm /tmp/tmp.DWAhJcTZOt /tmp/tmp.8cYesWfJ8a +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1716574003 300 +++ local start_time_sec=1716574003 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716574303 +++ local next_interval_min_edge=103 +++ local next_backup_at_sec=1716574200 +++ echo 1716574200 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 18:10:19 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 18:10:19 GMT ' --utc +%s +++ echo 1716574219 ++ local result=-19 ++ [[ -19 -gt 0 ]] ++++ get_current_epoch_time +++++ curl -sI https://percona.com/ +++++ grep date +++++ /usr/bin/sed 's/date: //g' ++++ local 'curent_date=Fri, 24 May 2024 18:10:19 GMT ' +++++ /usr/bin/date --date 'Fri, 24 May 2024 18:10:19 GMT ' --utc +%s ++++ echo 1716574219 +++ calculate_next_backup_start 1716574219 300 +++ local start_time_sec=1716574219 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716574519 +++ local next_interval_min_edge=19 +++ local next_backup_at_sec=1716574500 +++ echo 1716574500 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 18:10:20 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 18:10:20 GMT ' --utc +%s +++ echo 1716574220 ++ result=280 ++ [[ 280 -gt 0 ]] ++ echo 280 + sleep 280 + kubectl_bin wait job/some-name-16172-incr-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.kQvell2i6B ++ mktemp + local LAST_ERR=/tmp/tmp.khCL2SwSbO + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl wait job/some-name-16172-incr-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.kQvell2i6B job.batch/some-name-16172-incr-sch-backup condition met + cat /tmp/tmp.khCL2SwSbO + rm /tmp/tmp.kQvell2i6B /tmp/tmp.khCL2SwSbO + return 0 + restore some-name-16172 s3 incr + local cluster=some-name-16172 + local storage=s3 + local backup_type=incr + local restore_target= ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rv7Lozfldz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.DlKy1JlO03 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.rv7Lozfldz +++ cat /tmp/tmp.DlKy1JlO03 +++ rm /tmp/tmp.rv7Lozfldz /tmp/tmp.DlKy1JlO03 +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.lT2mxoL6Tm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.L8n9iFUoo1 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.lT2mxoL6Tm +++ cat /tmp/tmp.L8n9iFUoo1 +++ rm /tmp/tmp.lT2mxoL6Tm /tmp/tmp.L8n9iFUoo1 +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NNx91k9bfN ++++ mktemp +++ local LAST_ERR=/tmp/tmp.quNRfqb79H +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NNx91k9bfN +++ cat /tmp/tmp.quNRfqb79H +++ rm /tmp/tmp.NNx91k9bfN /tmp/tmp.quNRfqb79H +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.TlRgkbZNiB +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.AzqvWmWqaX ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.TlRgkbZNiB ++++ cat /tmp/tmp.AzqvWmWqaX ++++ rm /tmp/tmp.TlRgkbZNiB /tmp/tmp.AzqvWmWqaX ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TPb8JBEgr5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Wrg1Vnp9V8 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TPb8JBEgr5 +++ cat /tmp/tmp.Wrg1Vnp9V8 +++ rm /tmp/tmp.TPb8JBEgr5 /tmp/tmp.Wrg1Vnp9V8 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.rF1ESbMuKQ +++ mktemp ++ local LAST_ERR=/tmp/tmp.j6vE8rGfUn ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.rF1ESbMuKQ ++ cat /tmp/tmp.j6vE8rGfUn ++ rm /tmp/tmp.rF1ESbMuKQ /tmp/tmp.j6vE8rGfUn ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.7CDTg5DPIa ++ mktemp + local LAST_ERR=/tmp/tmp.ZdbMOS3xnZ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7CDTg5DPIa + cat /tmp/tmp.ZdbMOS3xnZ + rm /tmp/tmp.7CDTg5DPIa /tmp/tmp.ZdbMOS3xnZ + return 0 ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.XCjGeuA9Vo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sm5mJsfxuU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.XCjGeuA9Vo +++ cat /tmp/tmp.sm5mJsfxuU +++ rm /tmp/tmp.XCjGeuA9Vo /tmp/tmp.sm5mJsfxuU +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.daJGu78Q6q ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ST9Hmob4mj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.daJGu78Q6q +++ cat /tmp/tmp.ST9Hmob4mj +++ rm /tmp/tmp.daJGu78Q6q /tmp/tmp.ST9Hmob4mj +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8vTW3kdBEc ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2Bxyi6aaWL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.8vTW3kdBEc +++ cat /tmp/tmp.2Bxyi6aaWL +++ rm /tmp/tmp.8vTW3kdBEc /tmp/tmp.2Bxyi6aaWL +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.n91vMswDSC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.spi5wpaTkD ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.n91vMswDSC ++++ cat /tmp/tmp.spi5wpaTkD ++++ rm /tmp/tmp.n91vMswDSC /tmp/tmp.spi5wpaTkD ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZRQEvErGZ9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.31ZL1kBwSx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ZRQEvErGZ9 +++ cat /tmp/tmp.31ZL1kBwSx +++ rm /tmp/tmp.ZRQEvErGZ9 /tmp/tmp.31ZL1kBwSx +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-16172.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.lqmTaA0Hcw +++ mktemp ++ local LAST_ERR=/tmp/tmp.p1QrzdiHeO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.lqmTaA0Hcw ++ cat /tmp/tmp.p1QrzdiHeO ++ rm /tmp/tmp.lqmTaA0Hcw /tmp/tmp.p1QrzdiHeO ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.GObUGqDfxe ++ mktemp + local LAST_ERR=/tmp/tmp.XNY9QFwo3V + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.GObUGqDfxe + cat /tmp/tmp.XNY9QFwo3V + rm /tmp/tmp.GObUGqDfxe /tmp/tmp.XNY9QFwo3V + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vayky5QV8R ++++ mktemp +++ local LAST_ERR=/tmp/tmp.VXppkHLiqU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vayky5QV8R +++ cat /tmp/tmp.VXppkHLiqU +++ rm /tmp/tmp.vayky5QV8R /tmp/tmp.VXppkHLiqU +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.kMI0EAZ2G8 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.R35Smk0JDL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.kMI0EAZ2G8 +++ cat /tmp/tmp.R35Smk0JDL +++ rm /tmp/tmp.kMI0EAZ2G8 /tmp/tmp.R35Smk0JDL +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.8RaNxiZCJ5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.brzMLjLe6v +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.8RaNxiZCJ5 +++ cat /tmp/tmp.brzMLjLe6v +++ rm /tmp/tmp.8RaNxiZCJ5 /tmp/tmp.brzMLjLe6v +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.YwihNnhLNd +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.J1zzaQMhMz ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.YwihNnhLNd ++++ cat /tmp/tmp.J1zzaQMhMz ++++ rm /tmp/tmp.YwihNnhLNd /tmp/tmp.J1zzaQMhMz ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.iQvLyDrQQm ++++ mktemp +++ local LAST_ERR=/tmp/tmp.E5W3QzbuS3 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.iQvLyDrQQm +++ cat /tmp/tmp.E5W3QzbuS3 +++ rm /tmp/tmp.iQvLyDrQQm /tmp/tmp.E5W3QzbuS3 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.FSTUdLaLGa +++ mktemp ++ local LAST_ERR=/tmp/tmp.cBiGzhyHth ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.FSTUdLaLGa ++ cat /tmp/tmp.cBiGzhyHth ++ rm /tmp/tmp.FSTUdLaLGa /tmp/tmp.cBiGzhyHth ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.vv3P9JITZT ++ mktemp + local LAST_ERR=/tmp/tmp.fPwRI6w04r + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.vv3P9JITZT + cat /tmp/tmp.fPwRI6w04r + rm /tmp/tmp.vv3P9JITZT /tmp/tmp.fPwRI6w04r + return 0 ++ awk '{$1=$1;print}' +++ get_psql_user_pass postgres some-name-16172 +++ local user=postgres +++ local cluster=some-name-16172 +++ local namespace=scheduled-backup-13036 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.OhmpqO0aXa +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.xDDA52sv3s ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.OhmpqO0aXa ++++ cat /tmp/tmp.xDDA52sv3s ++++ rm /tmp/tmp.OhmpqO0aXa /tmp/tmp.xDDA52sv3s ++++ return 0 +++ [[ xsome-name-16172 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.hWhghuvKvl +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.WK6BceoCpf ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.hWhghuvKvl ++++ cat /tmp/tmp.WK6BceoCpf ++++ rm /tmp/tmp.hWhghuvKvl /tmp/tmp.WK6BceoCpf ++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.4DXE4BWvO8 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.GcNGfEx1lm ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.4DXE4BWvO8 ++++ cat /tmp/tmp.GcNGfEx1lm ++++ rm /tmp/tmp.4DXE4BWvO8 /tmp/tmp.GcNGfEx1lm ++++ return 0 +++ [[ xsome-name-16172-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.c2RyPyXonv ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.Gin8C8gncr +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.c2RyPyXonv +++++ cat /tmp/tmp.Gin8C8gncr +++++ rm /tmp/tmp.c2RyPyXonv /tmp/tmp.Gin8C8gncr +++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.9uYq8BAOs4 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.SwtwDXObVn ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.9uYq8BAOs4 ++++ cat /tmp/tmp.SwtwDXObVn ++++ rm /tmp/tmp.9uYq8BAOs4 /tmp/tmp.SwtwDXObVn ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-16172.scheduled-backup-13036 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EyDmXoCrOU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.N0DNxw4Dll +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.EyDmXoCrOU +++ cat /tmp/tmp.N0DNxw4Dll +++ rm /tmp/tmp.EyDmXoCrOU /tmp/tmp.N0DNxw4Dll +++ return 0 ++ local client_container=pg-client-76df9df876-pthwj ++ kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.9Fb6Keg3gW +++ mktemp ++ local LAST_ERR=/tmp/tmp.TpJvWg9Kwl ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.9Fb6Keg3gW ++ cat /tmp/tmp.TpJvWg9Kwl ++ rm /tmp/tmp.9Fb6Keg3gW /tmp/tmp.TpJvWg9Kwl ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bpuszHqHUz ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ug5QYYFVoc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bpuszHqHUz +++ cat /tmp/tmp.ug5QYYFVoc +++ rm /tmp/tmp.bpuszHqHUz /tmp/tmp.ug5QYYFVoc +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.JJ2ZeUk26c ++++ mktemp +++ local LAST_ERR=/tmp/tmp.tL8vBKi0Ek +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.JJ2ZeUk26c +++ cat /tmp/tmp.tL8vBKi0Ek +++ rm /tmp/tmp.JJ2ZeUk26c /tmp/tmp.tL8vBKi0Ek +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.EYC9vKT6K9 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9CSJg0Yo8s +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.EYC9vKT6K9 +++ cat /tmp/tmp.9CSJg0Yo8s +++ rm /tmp/tmp.EYC9vKT6K9 /tmp/tmp.9CSJg0Yo8s +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.aN502J5aWP +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.wrjCV6F17M ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.aN502J5aWP ++++ cat /tmp/tmp.wrjCV6F17M ++++ rm /tmp/tmp.aN502J5aWP /tmp/tmp.wrjCV6F17M ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WHciiRWIIu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9M6rI0NQDG +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.WHciiRWIIu +++ cat /tmp/tmp.9M6rI0NQDG +++ rm /tmp/tmp.WHciiRWIIu /tmp/tmp.9M6rI0NQDG +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gqmaU0hpTM +++ mktemp ++ local LAST_ERR=/tmp/tmp.8pqgPHN2Dg ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gqmaU0hpTM ++ cat /tmp/tmp.8pqgPHN2Dg ++ rm /tmp/tmp.gqmaU0hpTM /tmp/tmp.8pqgPHN2Dg ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.7BEh3bQPn7 ++ mktemp + local LAST_ERR=/tmp/tmp.v0qIsZy3fR + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.7BEh3bQPn7 + cat /tmp/tmp.v0qIsZy3fR + rm /tmp/tmp.7BEh3bQPn7 /tmp/tmp.v0qIsZy3fR + return 0 + run_restore some-name-16172 incr s3 773 xid + local cluster=some-name-16172 + local rstr_prefix=incr + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-16172 + yq w - spec.name incr-some-name-16172 + yq w - spec.namespace scheduled-backup-13036 + yq w - spec.parameters.backrest-storage-type s3 + yq w - spec.parameters.backrest-restore-cluster some-name-16172 + yq w - metadata.name incr-some-name-16172 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.7cfNRxU4oI/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.7cfNRxU4oI/restore.pitr.yml /tmp/tmp.7cfNRxU4oI/restore.yml + kubectl apply -f /tmp/tmp.7cfNRxU4oI/restore.yml pgtask.pg.percona.com/incr-some-name-16172 created + wait_bootstrap_completeness some-name-16172 + local cluster=some-name-16172 + wait_job_completion some-name-16172-bootstrap + local job=some-name-16172-bootstrap + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' some-name-16172-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-16172-backrest-shared-repo + local name=some-name-16172-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-16172-backrest-shared-repo + wait_deployment some-name-16172 + local name=some-name-16172 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-16172. + wait_job_completion some-name-16172-stanza-create false + local job=some-name-16172-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' some-name-16172-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-16172 + local job=backrest-backup-some-name-16172 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-16172+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-16172 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-16172 ++ local user=postgres ++ local cluster=some-name-16172 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.blvdI8JSW6 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.lJS1UyRvAL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.blvdI8JSW6 +++ cat /tmp/tmp.lJS1UyRvAL +++ rm /tmp/tmp.blvdI8JSW6 /tmp/tmp.lJS1UyRvAL +++ return 0 ++ [[ xsome-name-16172 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.rJb1ITqw0j ++++ mktemp +++ local LAST_ERR=/tmp/tmp.wCLfHkhWGZ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-16172 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.rJb1ITqw0j +++ cat /tmp/tmp.wCLfHkhWGZ +++ rm /tmp/tmp.rJb1ITqw0j /tmp/tmp.wCLfHkhWGZ +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2NlaoeXMtL ++++ mktemp +++ local LAST_ERR=/tmp/tmp.vNyIsWpzLx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2NlaoeXMtL +++ cat /tmp/tmp.vNyIsWpzLx +++ rm /tmp/tmp.2NlaoeXMtL /tmp/tmp.vNyIsWpzLx +++ return 0 ++ [[ xsome-name-16172-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.cvfHstFzbz +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.yaxt7vbSvY ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-16172 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.cvfHstFzbz ++++ cat /tmp/tmp.yaxt7vbSvY ++++ rm /tmp/tmp.cvfHstFzbz /tmp/tmp.yaxt7vbSvY ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.NGrXYVSF3d ++++ mktemp +++ local LAST_ERR=/tmp/tmp.IlyLr5iFlc +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-16172-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.NGrXYVSF3d +++ cat /tmp/tmp.IlyLr5iFlc +++ rm /tmp/tmp.NGrXYVSF3d /tmp/tmp.IlyLr5iFlc +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-16172.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-16172.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.gyEcmohs4G +++ mktemp ++ local LAST_ERR=/tmp/tmp.EihgShfpcl ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.gyEcmohs4G ++ cat /tmp/tmp.EihgShfpcl ++ rm /tmp/tmp.gyEcmohs4G /tmp/tmp.EihgShfpcl ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.zKUpfmxl2V ++ mktemp + local LAST_ERR=/tmp/tmp.g6hz2vFwoG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-16172.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.zKUpfmxl2V + cat /tmp/tmp.g6hz2vFwoG + rm /tmp/tmp.zKUpfmxl2V /tmp/tmp.g6hz2vFwoG + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql + delete_cluster_gracefully some-name-16172 + local cluster=some-name-16172 + kubectl_bin delete perconapgcluster/some-name-16172 ++ mktemp + local LAST_OUT=/tmp/tmp.MvRLeqBIUS ++ mktemp + local LAST_ERR=/tmp/tmp.JO9vXkc5xp + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete perconapgcluster/some-name-16172 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.MvRLeqBIUS perconapgcluster.pg.percona.com "some-name-16172" deleted + cat /tmp/tmp.JO9vXkc5xp + rm /tmp/tmp.MvRLeqBIUS /tmp/tmp.JO9vXkc5xp + return 0 ++ kubectl_bin -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-16172,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.OOm14WiPGi +++ mktemp ++ local LAST_ERR=/tmp/tmp.548ZsBcs78 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get jobs --selector=pg-cluster=some-name-16172,pgrmdata=true -o 'jsonpath={.items[0].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.OOm14WiPGi ++ cat /tmp/tmp.548ZsBcs78 ++ rm /tmp/tmp.OOm14WiPGi /tmp/tmp.548ZsBcs78 ++ return 0 + rmjob_name=some-name-16172-rmdata-quxb + wait_job_completion some-name-16172-rmdata-quxb false + local job=some-name-16172-rmdata-quxb + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' some-name-16172-rmdata-quxb+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-16172-rmdata-quxb -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + desc 'Testing full backup/restore from s3, local' + set +o xtrace ----------------------------------------------------------------------------------- Testing full backup/restore from s3, local ----------------------------------------------------------------------------------- + cluster=some-name-1419 + apply_configs some-name-1419 + local cluster=some-name-1419 + set +x secret/some-name-1419-backrest-repo-config created + yq w - metadata.name some-name-1419-custom-config + kubectl_bin apply -f - + yq r /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/custom-config.yml ++ mktemp + local LAST_OUT=/tmp/tmp.4OIR5pDqTc ++ mktemp + local LAST_ERR=/tmp/tmp.fRv2vGUfgG + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.4OIR5pDqTc configmap/some-name-1419-custom-config created + cat /tmp/tmp.fRv2vGUfgG + rm /tmp/tmp.4OIR5pDqTc /tmp/tmp.fRv2vGUfgG + return 0 + spinup_pgcluster some-name-1419 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml local,s3 some-name-1419-custom-config false true full + local cluster=some-name-1419 + local config=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local backup=local,s3 + local custom_config=some-name-1419-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local pgoClientFile=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + local namespace=scheduled-backup-13036 + local root_via_pgbouncer= + desc 'create fresh PG cluster' + set +o xtrace ----------------------------------------------------------------------------------- create fresh PG cluster ----------------------------------------------------------------------------------- + create_user_secrets some-name-1419 scheduled-backup-13036 + local cluster=some-name-1419 + local namespace=scheduled-backup-13036 + yq w - metadata.name some-name-1419-users-secret + kubectl_bin -n scheduled-backup-13036 apply -f - + yq r -d0 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + yq w - stringData.some-name some-name_pass ++ mktemp + local LAST_OUT=/tmp/tmp.E8e6YmtvDM ++ mktemp + local LAST_ERR=/tmp/tmp.Ulu0pwkr7M + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.E8e6YmtvDM secret/some-name-1419-users-secret created + cat /tmp/tmp.Ulu0pwkr7M + rm /tmp/tmp.E8e6YmtvDM /tmp/tmp.Ulu0pwkr7M + return 0 + yq w - metadata.name some-name-1419-pmm-secret + yq r -d1 /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/users-secret.yaml + kubectl_bin -n scheduled-backup-13036 apply -f - ++ mktemp + local LAST_OUT=/tmp/tmp.tgsdBjfhsP ++ mktemp + local LAST_ERR=/tmp/tmp.iQFSw6J359 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.tgsdBjfhsP secret/some-name-1419-pmm-secret created + cat /tmp/tmp.iQFSw6J359 + rm /tmp/tmp.tgsdBjfhsP /tmp/tmp.iQFSw6J359 + return 0 + kubectl_bin -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UnRvaNdcns ++ mktemp + local LAST_ERR=/tmp/tmp.6b801RfegD + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/conf/client.yml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.UnRvaNdcns deployment.apps/pg-client unchanged + cat /tmp/tmp.6b801RfegD + rm /tmp/tmp.UnRvaNdcns /tmp/tmp.6b801RfegD + return 0 + apply_cluster /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml some-name-1419 local,s3 false false some-name-1419-custom-config false true full false + local path=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml + local name=some-name-1419 + local backup=local,s3 + local restore_from=false + local restore_opts=false + local custom_config=some-name-1419-custom-config + local podAntiAffinity=false + local schedule=true + local schedule_type=full + local tls_only=false + local root_via_pgbouncer= + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/deploy/cr.yaml metadata.annotations.current-primary some-name-1419 + yq w - metadata.labels.crunchy-pgha-scope some-name-1419 + yq w - metadata.labels.deployment-name some-name-1419 + yq w - spec.keepData false + yq w - spec.name some-name-1419 + yq w - metadata.labels.pg-cluster some-name-1419 + yq w - spec.pgBadger.image perconalab/percona-postgresql-operator:main-ppg14-pgbadger + yq w - metadata.name some-name-1419 + yq w - spec.pmm.image perconalab/pmm-client:2.41.2 + yq w - spec.backup.backrestRepoImage perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo + yq w - spec.pgBouncer.image perconalab/percona-postgresql-operator:main-ppg14-pgbouncer + yq w - spec.backup.image perconalab/percona-postgresql-operator:main-ppg14-pgbackrest + yq w - spec.keepBackups false + yq w - spec.user some-name + yq w - spec.secretsName some-name-1419-users-secret + yq w - spec.pgPrimary.image perconalab/percona-postgresql-operator:main-ppg14-postgres-ha + yq w - spec.database some-name-1419 + yq w - metadata.labels.name some-name-1419 + yq w - spec.clustername some-name-1419 + case ${podAntiAffinity} in + case ${backup} in + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].type' s3 + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + [[ true == \t\r\u\e ]] + yq w /tmp/tmp.7cfNRxU4oI/cr.yaml 'spec.backup.storages[my-s3].bucket' pg-operator-testing + yq w - 'spec.backup.storages[my-s3].endpointUrl' s3.amazonaws.com + yq w - 'spec.backup.storages[my-s3].region' us-east-1 + yq w - 'spec.backup.schedule[0].name' sch-backup + yq w - 'spec.backup.storages[my-s3].uriStyle' path + yq w - 'spec.backup.schedule[0].schedule' '*/5 * * * *' + yq w - 'spec.backup.schedule[0].type' full + yq w - 'spec.backup.schedule[0].storage' my-s3 + yq w - 'spec.backup.storages[my-s3].type' s3 + mv /tmp/tmp.7cfNRxU4oI/cr.backup.yaml /tmp/tmp.7cfNRxU4oI/cr.yaml + [[ false != \f\a\l\s\e ]] + [[ some-name-1419-custom-config != \f\a\l\s\e ]] + yq w -i /tmp/tmp.7cfNRxU4oI/cr.yaml spec.pgPrimary.customconfig some-name-1419-custom-config + [[ false == \t\r\u\e ]] + [[ '' == \t\r\u\e ]] + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 ++ mktemp + local LAST_OUT=/tmp/tmp.W282oNhVkK ++ mktemp + local LAST_ERR=/tmp/tmp.cDehuyI2hC + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/cr.yaml -n scheduled-backup-13036 + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.W282oNhVkK perconapgcluster.pg.percona.com/some-name-1419 created + cat /tmp/tmp.cDehuyI2hC + rm /tmp/tmp.W282oNhVkK /tmp/tmp.cDehuyI2hC + return 0 + wait_deployment some-name-1419-backrest-shared-repo + local name=some-name-1419-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-1419-backrest-shared-repo. + wait_deployment some-name-1419 + local name=some-name-1419 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-1419...... + [[ local,s3 == \g\c\s\+* ]] + wait_job_completion some-name-1419-stanza-create false + local job=some-name-1419-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' some-name-1419-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_job_completion backrest-backup-some-name-1419 + local job=backrest-backup-some-name-1419 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-1419+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_cluster_consistency some-name-1419 + cluster_name=some-name-1419 + wait_cluster_status some-name-1419 'pgcluster Initialized' + local cluster_name=some-name-1419 + local 'target_status=pgcluster Initialized' + local target_namespace=scheduled-backup-13036 + local status_query= + case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.w65hyFoqNB +++ mktemp ++ local LAST_ERR=/tmp/tmp.bgHn9VN79o ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.w65hyFoqNB ++ cat /tmp/tmp.bgHn9VN79o ++ rm /tmp/tmp.w65hyFoqNB /tmp/tmp.bgHn9VN79o ++ return 0 + status_query='kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o jsonpath='\''{.status.PGCluster.state}'\''' + local retry=0 ++ eval kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath='\''{.status.PGCluster.state}'\''' +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.status.PGCluster.state}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.hw2lWpnKX5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bgVi5t9iA4 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.status.PGCluster.state}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.hw2lWpnKX5 +++ cat /tmp/tmp.bgVi5t9iA4 +++ rm /tmp/tmp.hw2lWpnKX5 /tmp/tmp.bgVi5t9iA4 +++ return 0 + [[ pgcluster Initialized == \p\g\c\l\u\s\t\e\r\ \I\n\i\t\i\a\l\i\z\e\d ]] + sleep 10 + desc 'write data' + set +o xtrace ----------------------------------------------------------------------------------- write data ----------------------------------------------------------------------------------- ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vdM0XE4L8G ++++ mktemp +++ local LAST_ERR=/tmp/tmp.m8bw7AJrdv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vdM0XE4L8G +++ cat /tmp/tmp.m8bw7AJrdv +++ rm /tmp/tmp.vdM0XE4L8G /tmp/tmp.m8bw7AJrdv +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.BSQsceCTRR ++++ mktemp +++ local LAST_ERR=/tmp/tmp.N3OF5YwAOx +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.BSQsceCTRR +++ cat /tmp/tmp.N3OF5YwAOx +++ rm /tmp/tmp.BSQsceCTRR /tmp/tmp.N3OF5YwAOx +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.qBayk1bhNt ++++ mktemp +++ local LAST_ERR=/tmp/tmp.6MdlDVLuTX +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.qBayk1bhNt +++ cat /tmp/tmp.6MdlDVLuTX +++ rm /tmp/tmp.qBayk1bhNt /tmp/tmp.6MdlDVLuTX +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.1jHwf8bU76 +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.CDHTAG1I4u ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.1jHwf8bU76 ++++ cat /tmp/tmp.CDHTAG1I4u ++++ rm /tmp/tmp.1jHwf8bU76 /tmp/tmp.CDHTAG1I4u ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.mjYP3wBppF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.V2ExShchDF +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.mjYP3wBppF +++ cat /tmp/tmp.V2ExShchDF +++ rm /tmp/tmp.mjYP3wBppF /tmp/tmp.V2ExShchDF +++ return 0 ++ echo postgres_password + run_psql 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.8uhulRFtsq +++ mktemp ++ local LAST_ERR=/tmp/tmp.3uzlQy7JXt ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.8uhulRFtsq ++ cat /tmp/tmp.3uzlQy7JXt ++ rm /tmp/tmp.8uhulRFtsq /tmp/tmp.3uzlQy7JXt ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.kHnMz7dY8u ++ mktemp + local LAST_ERR=/tmp/tmp.UzfQU4yh58 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.kHnMz7dY8u + cat /tmp/tmp.UzfQU4yh58 + rm /tmp/tmp.kHnMz7dY8u /tmp/tmp.UzfQU4yh58 + return 0 ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.99vaoa0ojr ++++ mktemp +++ local LAST_ERR=/tmp/tmp.y1ztH6GOUv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.99vaoa0ojr +++ cat /tmp/tmp.y1ztH6GOUv +++ rm /tmp/tmp.99vaoa0ojr /tmp/tmp.y1ztH6GOUv +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.3AxUNFdu0r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.qlZCe35eYA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.3AxUNFdu0r +++ cat /tmp/tmp.qlZCe35eYA +++ rm /tmp/tmp.3AxUNFdu0r /tmp/tmp.qlZCe35eYA +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.b5up0Ihr29 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.CjSvZ1ZxIj +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.b5up0Ihr29 +++ cat /tmp/tmp.CjSvZ1ZxIj +++ rm /tmp/tmp.b5up0Ihr29 /tmp/tmp.CjSvZ1ZxIj +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.gZfMa5HfTG +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.q3uA6cjEM5 ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.gZfMa5HfTG ++++ cat /tmp/tmp.q3uA6cjEM5 ++++ rm /tmp/tmp.gZfMa5HfTG /tmp/tmp.q3uA6cjEM5 ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Uj3bvjsWJS ++++ mktemp +++ local LAST_ERR=/tmp/tmp.WF7jPwkr6X +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Uj3bvjsWJS +++ cat /tmp/tmp.WF7jPwkr6X +++ rm /tmp/tmp.Uj3bvjsWJS /tmp/tmp.WF7jPwkr6X +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.qRtePB91Uw +++ mktemp ++ local LAST_ERR=/tmp/tmp.n3NQavt7jk ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.qRtePB91Uw ++ cat /tmp/tmp.n3NQavt7jk ++ rm /tmp/tmp.qRtePB91Uw /tmp/tmp.n3NQavt7jk ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.eJxunbJeAg ++ mktemp + local LAST_ERR=/tmp/tmp.EpoxEB3RC3 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.eJxunbJeAg + cat /tmp/tmp.EpoxEB3RC3 + rm /tmp/tmp.eJxunbJeAg /tmp/tmp.EpoxEB3RC3 + return 0 ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HViGmpKD7r ++++ mktemp +++ local LAST_ERR=/tmp/tmp.5ujKrr6xrC +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.HViGmpKD7r +++ cat /tmp/tmp.5ujKrr6xrC +++ rm /tmp/tmp.HViGmpKD7r /tmp/tmp.5ujKrr6xrC +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n90d97aTOW ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZjGa24SBEC +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.n90d97aTOW +++ cat /tmp/tmp.ZjGa24SBEC +++ rm /tmp/tmp.n90d97aTOW /tmp/tmp.ZjGa24SBEC +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.vBR6X4GpCw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9BPPRolati +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.vBR6X4GpCw +++ cat /tmp/tmp.9BPPRolati +++ rm /tmp/tmp.vBR6X4GpCw /tmp/tmp.9BPPRolati +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ejd8bsv4YU +++++ mktemp +++ base64 -d ++++ local LAST_ERR=/tmp/tmp.l5mRAbPLkn ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ejd8bsv4YU ++++ cat /tmp/tmp.l5mRAbPLkn ++++ rm /tmp/tmp.ejd8bsv4YU /tmp/tmp.l5mRAbPLkn ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.FVT7mqHi3W ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Hel24ySSp0 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.FVT7mqHi3W +++ cat /tmp/tmp.Hel24ySSp0 +++ rm /tmp/tmp.FVT7mqHi3W /tmp/tmp.Hel24ySSp0 +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.VcfjXcgvWP +++ mktemp ++ local LAST_ERR=/tmp/tmp.bJuBkikuo8 ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.VcfjXcgvWP ++ cat /tmp/tmp.bJuBkikuo8 ++ rm /tmp/tmp.VcfjXcgvWP /tmp/tmp.bJuBkikuo8 ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.pCOkX3p9pV ++ mktemp + local LAST_ERR=/tmp/tmp.LPO3e7mXBB + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ GRANT SELECT,INSERT ON myApp to "some-name";GRANT USAGE ON SCHEMA public TO "some-name";\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.pCOkX3p9pV + cat /tmp/tmp.LPO3e7mXBB + rm /tmp/tmp.pCOkX3p9pV /tmp/tmp.LPO3e7mXBB + return 0 + sleep 10 +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.backup.schedule[0].name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ljhsECk6KD ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZNR3UbzVmK +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.backup.schedule[0].name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ljhsECk6KD +++ cat /tmp/tmp.ZNR3UbzVmK +++ rm /tmp/tmp.ljhsECk6KD /tmp/tmp.ZNR3UbzVmK +++ return 0 ++ next_backup_after_sec some-name-1419-sch-backup ++ local conf_map_name=some-name-1419-sch-backup ++++ kubectl_bin get configmap/some-name-1419-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.dQrMFK8ViR +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8k75xfVhAl ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get configmap/some-name-1419-sch-backup -o 'jsonpath={.metadata.creationTimestamp}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.dQrMFK8ViR ++++ cat /tmp/tmp.8k75xfVhAl ++++ rm /tmp/tmp.dQrMFK8ViR /tmp/tmp.8k75xfVhAl ++++ return 0 +++ /usr/bin/date +%s --date=2024-05-24T18:21:16Z --utc ++ local configmap_creation_time=1716574876 +++ kubectl_bin get configmap/some-name-1419-sch-backup -o 'jsonpath={.data.schedule}' +++ jq .schedule +++ grep -Eo '[0-9]+' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Eut6iGiTf7 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.iBoCuowMwn +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get configmap/some-name-1419-sch-backup -o 'jsonpath={.data.schedule}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Eut6iGiTf7 +++ cat /tmp/tmp.iBoCuowMwn +++ rm /tmp/tmp.Eut6iGiTf7 /tmp/tmp.iBoCuowMwn +++ return 0 ++ local backup_time_interval_sec=300 +++ calculate_next_backup_start 1716574876 300 +++ local start_time_sec=1716574876 +++ local backup_time_interval_sec=300 +++ local next_interval_hop=1716575176 +++ local next_interval_min_edge=76 +++ local next_backup_at_sec=1716575100 +++ echo 1716575100 +++ get_current_epoch_time ++++ curl -sI https://percona.com/ ++++ grep date ++++ /usr/bin/sed 's/date: //g' +++ local 'curent_date=Fri, 24 May 2024 18:24:54 GMT ' ++++ /usr/bin/date --date 'Fri, 24 May 2024 18:24:54 GMT ' --utc +%s +++ echo 1716575094 ++ local result=6 ++ [[ 6 -gt 0 ]] ++ echo 6 + sleep 6 + kubectl_bin wait job/some-name-1419-full-sch-backup --for=condition=Complete --timeout=300s ++ mktemp + local LAST_OUT=/tmp/tmp.Px4C58IFEd ++ mktemp + local LAST_ERR=/tmp/tmp.vtRPPM8Aj2 + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl wait job/some-name-1419-full-sch-backup --for=condition=Complete --timeout=300s + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.Px4C58IFEd job.batch/some-name-1419-full-sch-backup condition met + cat /tmp/tmp.vtRPPM8Aj2 + rm /tmp/tmp.Px4C58IFEd /tmp/tmp.vtRPPM8Aj2 + return 0 + restore some-name-1419 s3 full + local cluster=some-name-1419 + local storage=s3 + local backup_type=full + local restore_target= ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.6WrR1Ft29Z ++++ mktemp +++ local LAST_ERR=/tmp/tmp.YayUBkH1RM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.6WrR1Ft29Z +++ cat /tmp/tmp.YayUBkH1RM +++ rm /tmp/tmp.6WrR1Ft29Z /tmp/tmp.YayUBkH1RM +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.HZoJ9JjVBU ++++ mktemp +++ local LAST_ERR=/tmp/tmp.UxtSbrWLZh +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.HZoJ9JjVBU +++ cat /tmp/tmp.UxtSbrWLZh +++ rm /tmp/tmp.HZoJ9JjVBU /tmp/tmp.UxtSbrWLZh +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.C9P3CaggJ5 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.sSoTdaLJhz +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.C9P3CaggJ5 +++ cat /tmp/tmp.sSoTdaLJhz +++ rm /tmp/tmp.C9P3CaggJ5 /tmp/tmp.sSoTdaLJhz +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.LripFxHa7q +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.1PdHZbogVS ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.LripFxHa7q ++++ cat /tmp/tmp.1PdHZbogVS ++++ rm /tmp/tmp.LripFxHa7q /tmp/tmp.1PdHZbogVS ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.UJOFRjjuKk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ZQLR6KKScA +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.UJOFRjjuKk +++ cat /tmp/tmp.ZQLR6KKScA +++ rm /tmp/tmp.UJOFRjjuKk /tmp/tmp.ZQLR6KKScA +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.vR0RocAbvk +++ mktemp ++ local LAST_ERR=/tmp/tmp.g9gK7SmSeO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.vR0RocAbvk ++ cat /tmp/tmp.g9gK7SmSeO ++ rm /tmp/tmp.vR0RocAbvk /tmp/tmp.g9gK7SmSeO ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.qkr2ecdUAJ ++ mktemp + local LAST_ERR=/tmp/tmp.3qvFzMEu7l + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.qkr2ecdUAJ + cat /tmp/tmp.3qvFzMEu7l + rm /tmp/tmp.qkr2ecdUAJ /tmp/tmp.3qvFzMEu7l + return 0 ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.yJtsLgtpH1 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.l8HKRXSmti +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.yJtsLgtpH1 +++ cat /tmp/tmp.l8HKRXSmti +++ rm /tmp/tmp.yJtsLgtpH1 /tmp/tmp.l8HKRXSmti +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ybOYq12NrE ++++ mktemp +++ local LAST_ERR=/tmp/tmp.MovQFKF3LQ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ybOYq12NrE +++ cat /tmp/tmp.MovQFKF3LQ +++ rm /tmp/tmp.ybOYq12NrE /tmp/tmp.MovQFKF3LQ +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WEC9MXV192 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.kA2ssE1jeM +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.WEC9MXV192 +++ cat /tmp/tmp.kA2ssE1jeM +++ rm /tmp/tmp.WEC9MXV192 /tmp/tmp.kA2ssE1jeM +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.0syM1OqWNC +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.5wC3wkP0uo ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.0syM1OqWNC ++++ cat /tmp/tmp.5wC3wkP0uo ++++ rm /tmp/tmp.0syM1OqWNC /tmp/tmp.5wC3wkP0uo ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.TxmrttFguT ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2j9ibxb5oJ +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.TxmrttFguT +++ cat /tmp/tmp.2j9ibxb5oJ +++ rm /tmp/tmp.TxmrttFguT /tmp/tmp.2j9ibxb5oJ +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-1419.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.ErHYScdsFK +++ mktemp ++ local LAST_ERR=/tmp/tmp.igok2RIIhw ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.ErHYScdsFK ++ cat /tmp/tmp.igok2RIIhw ++ rm /tmp/tmp.ErHYScdsFK /tmp/tmp.igok2RIIhw ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.aKXcTVXe22 ++ mktemp + local LAST_ERR=/tmp/tmp.lJDB1npIeL + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.aKXcTVXe22 + cat /tmp/tmp.lJDB1npIeL + rm /tmp/tmp.aKXcTVXe22 /tmp/tmp.lJDB1npIeL + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.OkG1R2B8rf ++++ mktemp +++ local LAST_ERR=/tmp/tmp.PCzwS3iMB9 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.OkG1R2B8rf +++ cat /tmp/tmp.PCzwS3iMB9 +++ rm /tmp/tmp.OkG1R2B8rf /tmp/tmp.PCzwS3iMB9 +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.ZC15rjGArw ++++ mktemp +++ local LAST_ERR=/tmp/tmp.9sezK6qy1i +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.ZC15rjGArw +++ cat /tmp/tmp.9sezK6qy1i +++ rm /tmp/tmp.ZC15rjGArw /tmp/tmp.9sezK6qy1i +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.2AAEDxJ59D ++++ mktemp +++ local LAST_ERR=/tmp/tmp.Rt8qTSxYoR +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.2AAEDxJ59D +++ cat /tmp/tmp.Rt8qTSxYoR +++ rm /tmp/tmp.2AAEDxJ59D /tmp/tmp.Rt8qTSxYoR +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.N5YJOQTUZA +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.8CZsGTJcTv ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.N5YJOQTUZA ++++ cat /tmp/tmp.8CZsGTJcTv ++++ rm /tmp/tmp.N5YJOQTUZA /tmp/tmp.8CZsGTJcTv ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.CspKC0oXmo ++++ mktemp +++ local LAST_ERR=/tmp/tmp.gw5QBSVBfr +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.CspKC0oXmo +++ cat /tmp/tmp.gw5QBSVBfr +++ rm /tmp/tmp.CspKC0oXmo /tmp/tmp.gw5QBSVBfr +++ return 0 ++ echo postgres_password + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.CuGr3nAnht +++ mktemp ++ local LAST_ERR=/tmp/tmp.lCtJS8EXpY ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.CuGr3nAnht ++ cat /tmp/tmp.lCtJS8EXpY ++ rm /tmp/tmp.CuGr3nAnht /tmp/tmp.lCtJS8EXpY ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.WeAsRxIsuF ++ mktemp + local LAST_ERR=/tmp/tmp.r0GKWKUENP + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.WeAsRxIsuF + cat /tmp/tmp.r0GKWKUENP + rm /tmp/tmp.WeAsRxIsuF /tmp/tmp.r0GKWKUENP + return 0 +++ get_psql_user_pass postgres some-name-1419 +++ local user=postgres +++ local cluster=some-name-1419 +++ local namespace=scheduled-backup-13036 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.7HRZG0f9iW +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.IwPTp0z1jz ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++ awk '{$1=$1;print}' ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.7HRZG0f9iW ++++ cat /tmp/tmp.IwPTp0z1jz ++++ rm /tmp/tmp.7HRZG0f9iW /tmp/tmp.IwPTp0z1jz ++++ return 0 +++ [[ xsome-name-1419 == \x ]] +++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in ++++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.2AKN8CmYtd +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.FQ5krqpQoZ ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.2AKN8CmYtd ++++ cat /tmp/tmp.FQ5krqpQoZ ++++ rm /tmp/tmp.2AKN8CmYtd /tmp/tmp.FQ5krqpQoZ ++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.Qcb4NXutYN +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.Utt47sNkJP ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.Qcb4NXutYN ++++ cat /tmp/tmp.Utt47sNkJP ++++ rm /tmp/tmp.Qcb4NXutYN /tmp/tmp.Utt47sNkJP ++++ return 0 +++ [[ xsome-name-1419-users-secret == \x ]] ++++ base64 -d +++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++++ mktemp +++++ local LAST_OUT=/tmp/tmp.UvUz25EaQr ++++++ mktemp +++++ local LAST_ERR=/tmp/tmp.q7GaTeWHKE +++++ local exit_status=0 +++++ local timeout=4 +++++ set +o errexit ++++++ seq 0 2 +++++ for i in '$(seq 0 2)' +++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ exit_status=0 +++++ [[ 0 != 0 ]] +++++ break +++++ set -o errexit +++++ cat /tmp/tmp.UvUz25EaQr +++++ cat /tmp/tmp.q7GaTeWHKE +++++ rm /tmp/tmp.UvUz25EaQr /tmp/tmp.q7GaTeWHKE +++++ return 0 ++++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.A7q0DI7opD +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.gXchys0HRk ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.A7q0DI7opD ++++ cat /tmp/tmp.gXchys0HRk ++++ rm /tmp/tmp.A7q0DI7opD /tmp/tmp.gXchys0HRk ++++ return 0 +++ echo postgres_password ++ run_psql '\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' postgres:postgres_password@some-name-1419.scheduled-backup-13036 ++ local 'command=\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;' ++ local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 ++ local driver=postgres ++ local suffix=.svc.cluster.local +++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.DxIKu5TnhF ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2GAEutMSAm +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.DxIKu5TnhF +++ cat /tmp/tmp.2GAEutMSAm +++ rm /tmp/tmp.DxIKu5TnhF /tmp/tmp.2GAEutMSAm +++ return 0 ++ local client_container=pg-client-76df9df876-pthwj ++ kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' +++ mktemp ++ local LAST_OUT=/tmp/tmp.xn0O3VcXO7 +++ mktemp ++ local LAST_ERR=/tmp/tmp.qrDKOIot2I ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT xmin FROM myApp WHERE id=100501;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.xn0O3VcXO7 ++ cat /tmp/tmp.qrDKOIot2I ++ rm /tmp/tmp.xn0O3VcXO7 /tmp/tmp.qrDKOIot2I ++ return 0 + restore_target=773 ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.c4GLAzdO4X ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2ZZMibdNWI +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.c4GLAzdO4X +++ cat /tmp/tmp.2ZZMibdNWI +++ rm /tmp/tmp.c4GLAzdO4X /tmp/tmp.2ZZMibdNWI +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.btV0Ko0Dyk ++++ mktemp +++ local LAST_ERR=/tmp/tmp.bgIAdXtjqV +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.btV0Ko0Dyk +++ cat /tmp/tmp.bgIAdXtjqV +++ rm /tmp/tmp.btV0Ko0Dyk /tmp/tmp.bgIAdXtjqV +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.bgJHKmqDKC ++++ mktemp +++ local LAST_ERR=/tmp/tmp.2ulTsP4GuU +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.bgJHKmqDKC +++ cat /tmp/tmp.2ulTsP4GuU +++ rm /tmp/tmp.bgJHKmqDKC /tmp/tmp.2ulTsP4GuU +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.ekqEzJBayr +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.s42W3eIvIJ ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.ekqEzJBayr ++++ cat /tmp/tmp.s42W3eIvIJ ++++ rm /tmp/tmp.ekqEzJBayr /tmp/tmp.s42W3eIvIJ ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.Yzs7RA0ATy ++++ mktemp +++ local LAST_ERR=/tmp/tmp.U2LNZUFJau +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.Yzs7RA0ATy +++ cat /tmp/tmp.U2LNZUFJau +++ rm /tmp/tmp.Yzs7RA0ATy /tmp/tmp.U2LNZUFJau +++ return 0 ++ echo postgres_password + run_psql 'DROP DATABASE myapp' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local 'command=DROP DATABASE myapp' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.f8blGsx7NA +++ mktemp ++ local LAST_ERR=/tmp/tmp.BhkDGoIqlV ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.f8blGsx7NA ++ cat /tmp/tmp.BhkDGoIqlV ++ rm /tmp/tmp.f8blGsx7NA /tmp/tmp.BhkDGoIqlV ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.yBwImG2bH9 ++ mktemp + local LAST_ERR=/tmp/tmp.190T0Z5GEw + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''DROP DATABASE myapp\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.yBwImG2bH9 + cat /tmp/tmp.190T0Z5GEw + rm /tmp/tmp.yBwImG2bH9 /tmp/tmp.190T0Z5GEw + return 0 + run_restore some-name-1419 full s3 773 xid + local cluster=some-name-1419 + local rstr_prefix=full + local storage=s3 + local target=773 + local type=xid + yq w /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/conf/restore.yml metadata.labels.pg-cluster some-name-1419 + yq w - spec.name full-some-name-1419 + yq w - metadata.name full-some-name-1419 + yq w - spec.namespace scheduled-backup-13036 + yq w - spec.parameters.backrest-restore-cluster some-name-1419 + yq w - spec.parameters.backrest-storage-type s3 + [[ 773 != \n\u\l\l ]] + yq w /tmp/tmp.7cfNRxU4oI/restore.yml --style=single spec.parameters.backrest-pitr-target 773 + yq w --style=single -- - spec.parameters.backrest-restore-opts --type=xid + mv /tmp/tmp.7cfNRxU4oI/restore.pitr.yml /tmp/tmp.7cfNRxU4oI/restore.yml + kubectl apply -f /tmp/tmp.7cfNRxU4oI/restore.yml pgtask.pg.percona.com/full-some-name-1419 created + wait_bootstrap_completeness some-name-1419 + local cluster=some-name-1419 + wait_job_completion some-name-1419-bootstrap + local job=some-name-1419-bootstrap + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 1 -ge 120 ]] + [[ 1 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 2 -ge 120 ]] + [[ 2 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + sleep 1 + echo -n . .+ let retry+=1 + [[ 3 -ge 120 ]] + [[ 3 -ge 120 ]] + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' some-name-1419-bootstrap+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-bootstrap -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ not found == \n\o\t\ \f\o\u\n\d ]] + echo + wait_deployment some-name-1419-backrest-shared-repo + local name=some-name-1419-backrest-shared-repo + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-1419-backrest-shared-repo... + wait_deployment some-name-1419 + local name=some-name-1419 + local target_namespace=scheduled-backup-13036 + sleep 10 + set +o xtrace some-name-1419.... + wait_job_completion some-name-1419-stanza-create false + local job=some-name-1419-stanza-create + local mandatory=false + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' some-name-1419-stanza-create+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/some-name-1419-stanza-create -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + wait_job_completion backrest-backup-some-name-1419 + local job=backrest-backup-some-name-1419 + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' backrest-backup-some-name-1419+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 30 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 31 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 32 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 33 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 34 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 35 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 36 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 37 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 38 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 39 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 40 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 41 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 42 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 43 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 44 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 45 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 46 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 47 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 48 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 49 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 50 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 51 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 52 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 53 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 54 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 55 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 56 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/backrest-backup-some-name-1419 -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo ++ get_psql_user_pass postgres some-name-1419 ++ local user=postgres ++ local cluster=some-name-1419 ++ local namespace=scheduled-backup-13036 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.RQGRltwt59 ++++ mktemp +++ local LAST_ERR=/tmp/tmp.ppNFa7bFtd +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.name}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.RQGRltwt59 +++ cat /tmp/tmp.ppNFa7bFtd +++ rm /tmp/tmp.RQGRltwt59 /tmp/tmp.ppNFa7bFtd +++ return 0 ++ [[ xsome-name-1419 == \x ]] ++ case $(kubectl_bin get perconapgcluster/${cluster_name} -o jsonpath='{.metadata.labels.pgo-version}') in +++ kubectl_bin get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.WoRpMYRcGx ++++ mktemp +++ local LAST_ERR=/tmp/tmp.RPnTQLBPIv +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl get perconapgcluster/some-name-1419 -o 'jsonpath={.metadata.labels.pgo-version}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.WoRpMYRcGx +++ cat /tmp/tmp.RPnTQLBPIv +++ rm /tmp/tmp.WoRpMYRcGx /tmp/tmp.RPnTQLBPIv +++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.jBVR7rgWaZ ++++ mktemp +++ local LAST_ERR=/tmp/tmp.oVX1hTFcf7 +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.jBVR7rgWaZ +++ cat /tmp/tmp.oVX1hTFcf7 +++ rm /tmp/tmp.jBVR7rgWaZ /tmp/tmp.oVX1hTFcf7 +++ return 0 ++ [[ xsome-name-1419-users-secret == \x ]] +++ base64 -d ++++ kubectl_bin -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' +++++ mktemp ++++ local LAST_OUT=/tmp/tmp.pXap5y9EUL +++++ mktemp ++++ local LAST_ERR=/tmp/tmp.9iZ7pRiXhV ++++ local exit_status=0 ++++ local timeout=4 ++++ set +o errexit +++++ seq 0 2 ++++ for i in '$(seq 0 2)' ++++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++++ kubectl -n scheduled-backup-13036 get perconapgcluster/some-name-1419 -o 'jsonpath={.spec.secretsName}' ++++ exit_status=0 ++++ [[ 0 != 0 ]] ++++ break ++++ set -o errexit ++++ cat /tmp/tmp.pXap5y9EUL ++++ cat /tmp/tmp.9iZ7pRiXhV ++++ rm /tmp/tmp.pXap5y9EUL /tmp/tmp.9iZ7pRiXhV ++++ return 0 +++ kubectl_bin -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' ++++ mktemp +++ local LAST_OUT=/tmp/tmp.n63ZFuVyDu ++++ mktemp +++ local LAST_ERR=/tmp/tmp.cA1S0zRKsL +++ local exit_status=0 +++ local timeout=4 +++ set +o errexit ++++ seq 0 2 +++ for i in '$(seq 0 2)' +++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup +++ kubectl -n scheduled-backup-13036 get secret/some-name-1419-users-secret -o 'jsonpath={.data.postgres}' +++ exit_status=0 +++ [[ 0 != 0 ]] +++ break +++ set -o errexit +++ cat /tmp/tmp.n63ZFuVyDu +++ cat /tmp/tmp.cA1S0zRKsL +++ rm /tmp/tmp.n63ZFuVyDu /tmp/tmp.cA1S0zRKsL +++ return 0 ++ echo postgres_password + compare_psql_cmd select-1 '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local command_id=select-1 + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local postfix= + local uri_suffix= + local expected_result=/mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql + run_psql '\c myapp \\\ SELECT * from myApp;' postgres:postgres_password@some-name-1419.scheduled-backup-13036 postgres '' + local 'command=\c myapp \\\ SELECT * from myApp;' + local uri=postgres:postgres_password@some-name-1419.scheduled-backup-13036 + local driver=postgres + local suffix=.svc.cluster.local ++ kubectl_bin -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' +++ mktemp ++ local LAST_OUT=/tmp/tmp.k3XMmMQIRi +++ mktemp ++ local LAST_ERR=/tmp/tmp.lsFUgjn4UB ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl -n scheduled-backup-13036 get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.k3XMmMQIRi ++ cat /tmp/tmp.lsFUgjn4UB ++ rm /tmp/tmp.k3XMmMQIRi /tmp/tmp.lsFUgjn4UB ++ return 0 + local client_container=pg-client-76df9df876-pthwj + kubectl_bin exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' ++ mktemp + local LAST_OUT=/tmp/tmp.IahCvyDzii ++ mktemp + local LAST_ERR=/tmp/tmp.2oTROGsXDe + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl exec pg-client-76df9df876-pthwj -n scheduled-backup-13036 -- bash -c 'printf '\''\c myapp \\\ SELECT * from myApp;\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:postgres_password@some-name-1419.scheduled-backup-13036.svc.cluster.local' + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.IahCvyDzii + cat /tmp/tmp.2oTROGsXDe + rm /tmp/tmp.IahCvyDzii /tmp/tmp.2oTROGsXDe + return 0 + '[' '!' -s /tmp/tmp.7cfNRxU4oI/select-1.sql ']' + diff -u /mnt/jenkins/workspace/cloud-pg-operator_PR-776/e2e-tests/scheduled-backup/compare/select-1.sql /tmp/tmp.7cfNRxU4oI/select-1.sql + destroy scheduled-backup-13036 + grep -v level=info + grep -v level=debug ++ get_operator_pod ++ kubectl_bin get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' + grep -v 'Getting pods from source' + grep -v 'Getting tasks for pod' + grep -v 'the object has been modified' + grep -v 'get backup status: Job.batch' +++ mktemp + /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' + sort -u ++ local LAST_OUT=/tmp/tmp.V35BLT7H3g + tee /tmp/tmp.7cfNRxU4oI/operator.log +++ mktemp ++ local LAST_ERR=/tmp/tmp.U76V8wwpBO ++ local exit_status=0 ++ local timeout=4 ++ set +o errexit +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup ++ kubectl get pods --selector=name=postgres-operator -o 'jsonpath={.items[].metadata.name}' ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ set -o errexit ++ cat /tmp/tmp.V35BLT7H3g ++ cat /tmp/tmp.U76V8wwpBO ++ rm /tmp/tmp.V35BLT7H3g /tmp/tmp.U76V8wwpBO ++ return 0 + kubectl_bin logs postgres-operator-57d977b5cc-hssbr -c operator ++ mktemp + local LAST_OUT=/tmp/tmp.jF9KkdBs8H ++ mktemp + local LAST_ERR=/tmp/tmp.NLGJm1n1ab + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl logs postgres-operator-57d977b5cc-hssbr -c operator + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.jF9KkdBs8H + cat /tmp/tmp.NLGJm1n1ab + rm /tmp/tmp.jF9KkdBs8H /tmp/tmp.NLGJm1n1ab + return 0 ] { } }, ] } }, { } }, ] { } }, { } ] { } }, { } }, ] } ] { } }, }, { ] { } }, ] ] ], { } }, }, { }, { },{ }] }], ] { } }, , ], { } }, }, { }], { } }, }], } ], } }, } }, } }, { } }, },{ }, 2024/05/24 17:39:52 INF 1 (localhost:4150) connecting to nsqd 2024/05/24 17:41:10 INF 2 (localhost:4150) connecting to nsqd 2024/05/24 17:41:11 INF 3 (localhost:4150) connecting to nsqd 2024/05/24 17:41:35 INF 4 (localhost:4150) connecting to nsqd 2024/05/24 17:43:31 INF 5 (localhost:4150) connecting to nsqd 2024/05/24 17:43:31 INF 6 (localhost:4150) connecting to nsqd 2024/05/24 17:43:31 INF 7 (localhost:4150) connecting to nsqd 2024/05/24 17:45:00 INF 8 (localhost:4150) connecting to nsqd 2024/05/24 17:47:04 INF 9 (localhost:4150) connecting to nsqd 2024/05/24 17:47:45 INF 10 (localhost:4150) connecting to nsqd 2024/05/24 17:48:34 INF 11 (localhost:4150) connecting to nsqd 2024/05/24 17:49:08 INF 12 (localhost:4150) connecting to nsqd 2024/05/24 17:49:26 INF 13 (localhost:4150) connecting to nsqd 2024/05/24 17:50:00 INF 14 (localhost:4150) connecting to nsqd 2024/05/24 17:51:32 INF 15 (localhost:4150) connecting to nsqd 2024/05/24 17:51:32 INF 16 (localhost:4150) connecting to nsqd 2024/05/24 17:51:33 INF 17 (localhost:4150) connecting to nsqd 2024/05/24 17:53:49 INF 18 (localhost:4150) connecting to nsqd 2024/05/24 17:53:55 INF 19 (localhost:4150) connecting to nsqd 2024/05/24 17:54:18 INF 20 (localhost:4150) connecting to nsqd 2024/05/24 17:54:23 INF 21 (localhost:4150) connecting to nsqd 2024/05/24 17:54:42 INF 22 (localhost:4150) connecting to nsqd 2024/05/24 17:55:37 INF 23 (localhost:4150) connecting to nsqd 2024/05/24 17:55:38 INF 24 (localhost:4150) connecting to nsqd 2024/05/24 17:55:46 INF 25 (localhost:4150) connecting to nsqd 2024/05/24 17:57:39 INF 26 (localhost:4150) connecting to nsqd 2024/05/24 17:57:40 INF 27 (localhost:4150) connecting to nsqd 2024/05/24 17:57:40 INF 28 (localhost:4150) connecting to nsqd 2024/05/24 18:00:00 INF 29 (localhost:4150) connecting to nsqd 2024/05/24 18:00:45 INF 30 (localhost:4150) connecting to nsqd 2024/05/24 18:01:29 INF 31 (localhost:4150) connecting to nsqd 2024/05/24 18:02:05 INF 32 (localhost:4150) connecting to nsqd 2024/05/24 18:02:45 INF 33 (localhost:4150) connecting to nsqd 2024/05/24 18:02:54 INF 34 (localhost:4150) connecting to nsqd 2024/05/24 18:04:58 INF 35 (localhost:4150) connecting to nsqd 2024/05/24 18:04:59 INF 36 (localhost:4150) connecting to nsqd 2024/05/24 18:04:59 INF 37 (localhost:4150) connecting to nsqd 2024/05/24 18:05:00 INF 38 (localhost:4150) connecting to nsqd 2024/05/24 18:05:10 INF 39 (localhost:4150) connecting to nsqd 2024/05/24 18:05:19 INF 40 (localhost:4150) connecting to nsqd 2024/05/24 18:06:05 INF 41 (localhost:4150) connecting to nsqd 2024/05/24 18:06:10 INF 42 (localhost:4150) connecting to nsqd 2024/05/24 18:06:34 INF 43 (localhost:4150) connecting to nsqd 2024/05/24 18:07:23 INF 44 (localhost:4150) connecting to nsqd 2024/05/24 18:07:23 INF 45 (localhost:4150) connecting to nsqd 2024/05/24 18:07:32 INF 46 (localhost:4150) connecting to nsqd 2024/05/24 18:09:29 INF 47 (localhost:4150) connecting to nsqd 2024/05/24 18:09:29 INF 48 (localhost:4150) connecting to nsqd 2024/05/24 18:09:29 INF 49 (localhost:4150) connecting to nsqd 2024/05/24 18:10:00 INF 50 (localhost:4150) connecting to nsqd 2024/05/24 18:10:45 INF 51 (localhost:4150) connecting to nsqd 2024/05/24 18:15:00 INF 52 (localhost:4150) connecting to nsqd 2024/05/24 18:15:18 INF 53 (localhost:4150) connecting to nsqd 2024/05/24 18:16:03 INF 54 (localhost:4150) connecting to nsqd 2024/05/24 18:16:44 INF 55 (localhost:4150) connecting to nsqd 2024/05/24 18:17:23 INF 56 (localhost:4150) connecting to nsqd 2024/05/24 18:17:32 INF 57 (localhost:4150) connecting to nsqd 2024/05/24 18:19:39 INF 58 (localhost:4150) connecting to nsqd 2024/05/24 18:19:40 INF 59 (localhost:4150) connecting to nsqd 2024/05/24 18:19:40 INF 60 (localhost:4150) connecting to nsqd 2024/05/24 18:19:50 INF 61 (localhost:4150) connecting to nsqd 2024/05/24 18:20:42 INF 62 (localhost:4150) connecting to nsqd 2024/05/24 18:20:47 INF 63 (localhost:4150) connecting to nsqd 2024/05/24 18:21:07 INF 64 (localhost:4150) connecting to nsqd 2024/05/24 18:21:56 INF 65 (localhost:4150) connecting to nsqd 2024/05/24 18:21:57 INF 66 (localhost:4150) connecting to nsqd 2024/05/24 18:22:05 INF 67 (localhost:4150) connecting to nsqd 2024/05/24 18:24:04 INF 68 (localhost:4150) connecting to nsqd 2024/05/24 18:24:04 INF 69 (localhost:4150) connecting to nsqd 2024/05/24 18:24:05 INF 70 (localhost:4150) connecting to nsqd 2024/05/24 18:25:00 INF 71 (localhost:4150) connecting to nsqd 2024/05/24 18:27:13 INF 72 (localhost:4150) connecting to nsqd 2024/05/24 18:27:56 INF 73 (localhost:4150) connecting to nsqd 2024/05/24 18:28:38 INF 74 (localhost:4150) connecting to nsqd 2024/05/24 18:29:41 INF 75 (localhost:4150) connecting to nsqd 2024/05/24 18:29:49 INF 76 (localhost:4150) connecting to nsqd 2024/05/24 18:30:00 INF 77 (localhost:4150) connecting to nsqd 2024/05/24 18:31:56 INF 78 (localhost:4150) connecting to nsqd 2024/05/24 18:31:56 INF 79 (localhost:4150) connecting to nsqd 2024/05/24 18:31:56 INF 80 (localhost:4150) connecting to nsqd "accessModes": [ "affinity": "affinity": { "affinity": { "podAntiAffinity": { "allowPrivilegeEscalation": false, "annotations": {"keep-backups":"false","keep-data":"false"}, "apiVersion": "apps/v1", "apiVersion": "batch/v1", "apiVersion": "v1", "apiVersion": "v1", "apiVersion": "v1", "backrest-command": "backup", "backrest-command": "stanza-create", "claimName": "some-name-11182-pgbr-repo" "claimName": "some-name-1419-pgbr-repo" "claimName": "some-name-16172-pgbr-repo" "claimName": "some-name-17978-pgbr-repo" "command": [ "configMap": { "configMap": { "configMap": { "configMapKeyRef": { "containerName": "database", "containerPort": 2022, "containerPort": 5432, "containerPort": 8009, "containers": [ "containers": [{ "cpu": "1", "cpu": "200m", "cpu": "500m", "crunchydata" "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgbouncer": "true", "crunchy-pgha-scope": "some-name-11182","pgo-version": "1.7.0","name": "some-name-11182","pg-cluster": "some-name-11182","deployment-name": "some-name-11182","pgouser": "admin" "crunchy-pgha-scope": "some-name-11182","pgo-version": "1.7.0","name": "some-name-11182","pg-cluster": "some-name-11182","deployment-name": "some-name-11182","pgouser": "admin" "crunchy-pgha-scope": "some-name-1419","pgo-version": "1.7.0","name": "some-name-1419","pg-cluster": "some-name-1419","deployment-name": "some-name-1419","pgouser": "admin" "crunchy-pgha-scope": "some-name-16172","pgo-version": "1.7.0","name": "some-name-16172","pg-cluster": "some-name-16172","deployment-name": "some-name-16172","pgouser": "admin" "crunchy-pgha-scope": "some-name-17978","pgo-version": "1.7.0","name": "some-name-17978","pg-cluster": "some-name-17978","deployment-name": "some-name-17978","pgouser": "admin" "defaultMode": 288 "defaultMode": 288, "defaultMode": 288, "defaultMode": 420, "deployment-name": "some-name-11182" "deployment-name": "some-name-11182","pgouser": "admin","crunchy-pgha-scope": "some-name-11182","pgo-version": "1.7.0","name": "some-name-11182","pg-cluster": "some-name-11182" "deployment-name": "some-name-11182-repl1" "deployment-name": "some-name-11182-repl1","pgo-version": "1.7.0","name": "some-name-11182-replica","pg-cluster": "some-name-11182" "deployment-name": "some-name-11182-repl2" "deployment-name": "some-name-11182-repl2","pgo-version": "1.7.0","name": "some-name-11182-replica","pg-cluster": "some-name-11182" "deployment-name": "some-name-1419" "deployment-name": "some-name-1419","pgouser": "admin","crunchy-pgha-scope": "some-name-1419","pgo-version": "1.7.0","name": "some-name-1419","pg-cluster": "some-name-1419" "deployment-name": "some-name-1419-repl1" "deployment-name": "some-name-1419-repl2" "deployment-name": "some-name-16172" "deployment-name": "some-name-16172-repl1" "deployment-name": "some-name-16172-repl2" "deployment-name": "some-name-16172-repl2","pgo-version": "1.7.0","name": "some-name-16172-replica","pg-cluster": "some-name-16172" "deployment-name": "some-name-17978" "deployment-name": "some-name-17978-repl1" "deployment-name": "some-name-17978-repl1","pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978" "deployment-name": "some-name-17978-repl1","pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978" "deployment-name": "some-name-17978-repl2" "deployment-name": "some-name-17978-repl2","pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978" "divisor": "1m", "dnsPolicy": "ClusterFirst" "downwardAPI": { "emptyDir": { "emptyDir": { "emptyDir": { "emptyDir": { "env": [ "env": [{ "exec": { "fieldPath": "metadata.annotations" "fieldPath": "metadata.labels" "fieldPath": "metadata.labels['crunchy-pgha-scope']" "fieldPath": "metadata.namespace" "fieldRef": { "fieldRef": { "fsGroup": 2, I0524 17:39:10.464258 1 shared_informer.go:240] Waiting for caches to sync for namespace I0524 17:39:10.564772 1 shared_informer.go:247] Caches are synced for namespace I0524 17:39:11.004002 1 shared_informer.go:240] Waiting for caches to sync for scheduled-backup-13036 I0524 17:39:11.104899 1 shared_informer.go:247] Caches are synced for scheduled-backup-13036 "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbackrest-repo", "image": "perconalab/percona-postgresql-operator:main-ppg14-pgbouncer", "image": "perconalab/percona-postgresql-operator:main-ppg14-postgres-ha", "image": "perconalab/percona-postgresql-operator:PR-776-0087ef129-pgo-rmdata", "imagePullPolicy": "IfNotPresent" "imagePullPolicy": "IfNotPresent", "initialDelaySeconds": 15 "initialDelaySeconds": 30, "items": [ "items": [ "items": [ "items": [ "key": "aws-s3-key" "key": "aws-s3-key-secret" "key": "config", "key": "config", "key": "config", "key": "crunchy-pgbouncer", "key": "init" "key": "password" "key": "pg-cluster", "key": "pgo-backrest", "key": "pgo-backrest-job", "key": "pgo-pg-database", "key": "pg-pod-anti-affinity", "key": "tls.crt", "key": "tls.key", "key": "vendor", "kind": "Deployment", "kind": "Job", "kind": "PersistentVolumeClaim", "kind": "Service", "labels": { "labels": { "labels": { "labelSelector": { "livenessProbe": { "matchExpressions": [ "matchLabels": { "matchLabels": { "maxSurge": 1 "maxUnavailable": 1, "medium": "Memory", "medium": "Memory" "medium": "Memory", "medium": "Memory", "medium": "Memory", "memory": "128Mi" "memory": "256Mi" "memory": "48Mi" "metadata": { "metadata": { "metadata": { "mountPath": "/backrestrepo", "mountPath": "/dev/shm", "mountPath": "/dev/shm", "mountPath": "/etc/pgbackrest" "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/pgbackrest/conf.d", "mountPath": "/etc/podinfo", "mountPath": "/etc/ssh", "mountPath": "/etc/ssh", "mountPath": "/home/pgbackrest" "mountPath": "/pgconf", "mountPath": "/pgconf/", "mountPath": "/pgconf", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgreplicator", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pgsuper", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/pguser", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls", "mountPath": "/pgconf/tls/pgbouncer", "mountPath": "/pgconf/tls-replication", "mountPath": "/pgdata", "mountPath": "/pgdata", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/sshd", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp", "mountPath": "/tmp" "name": "backrest", "name": "backrest-backup-some-name-11182", "name": "backrest-backup-some-name-1419", "name": "backrest-backup-some-name-16172", "name": "backrest-backup-some-name-17978", "name": "backrestrepo", "name": "backrestrepo", "name": "BACKREST_SKIP_CREATE_STANZA", "name": "cluster1-replication-ssl-keypair", "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-ca" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "cluster1-ssl-keypair" "name": "COMMAND", "name": "COMMAND_OPTS", "name": "database", "name": "dshm" "name": "dshm" "name": "dshm", "name": "dshm", "name": "ENABLE_SSHD", "name": "IS_BACKUP", "name": "IS_REPLICA", "name": "LD_PRELOAD", "name": "LD_PRELOAD", "name": "MODE", "name": "NAMESPACE", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_GROUP", "name": "NSS_WRAPPER_PASSWD", "name": "NSS_WRAPPER_PASSWD", "name": "patroni" "name": "PATRONI_KUBERNETES_LABELS", "name": "PATRONI_KUBERNETES_NAMESPACE", "name": "PATRONI_KUBERNETES_SCOPE_LABEL", "name": "PATRONI_LOG_LEVEL", "name": "PATRONI_POSTGRESQL_DATA_DIR", "name": "PATRONI_SCOPE", "name": "pgbackrest" "name": "pgbackrest-conf", "name": "pgbackrest-conf", "name": "pgbackrest-config" "name": "pgbackrest-config" "name": "pgbackrest-config", "name": "pgbackrest-config", "name": "PGBACKREST_DB_HOST", "name": "PGBACKREST_DB_PATH", "name": "PGBACKREST_DB_PATH", "name": "pgbackrest-home", "name": "pgbackrest-home", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_LOG_PATH", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_PORT", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_PG1_SOCKET_PATH", "name": "PGBACKREST_REPO1_HOST", "name": "PGBACKREST_REPO1_HOST_CMD", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_PATH", "name": "PGBACKREST_REPO1_S3_BUCKET", "name": "PGBACKREST_REPO1_S3_CA_FILE", "name": "PGBACKREST_REPO1_S3_ENDPOINT", "name": "PGBACKREST_REPO1_S3_KEY", "name": "PGBACKREST_REPO1_S3_KEY_SECRET", "name": "PGBACKREST_REPO1_S3_REGION", "name": "PGBACKREST_REPO1_S3_URI_STYLE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_REPO1_TYPE", "name": "PGBACKREST_STANZA", "name": "PGBACKREST_STANZA", "name": "pgbouncer" "name": "pgbouncer", "name": "pgbouncer-conf", "name": "pgbouncer-conf", "name": "PG_CLUSTER", "name": "pgconf-volume" "name": "pgconf-volume" "name": "pgconf-volume", "name": "pgconf-volume", "name": "pgdata", "name": "pgdata" "name": "pgdata", "name": "pgdata", "name": "PGHA_BOOTSTRAP_METHOD", "name": "PGHA_DATABASE", "name": "PGHA_INIT", "name": "PGHA_PASSWORD_TYPE", "name": "PGHA_PGBACKREST", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_GCS_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_LOCAL_S3_STORAGE", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PGBACKREST_S3_VERIFY_TLS", "name": "PGHA_PG_PORT", "name": "PGHA_REPLICA_REINIT_ON_START_FAIL", "name": "PGHA_SCOPE", "name": "PGHA_STANDBY", "name": "PGHA_SYNC_REPLICATION", "name": "PGHA_TLS_ENABLED", "name": "PGHA_TLS_ONLY", "name": "PGHA_USER", "name": "PGHOST", "name": "PG_PASSWORD", "name": "PG_PRIMARY_SERVICE_NAME", "name": "PITR_TARGET", "name": "podinfo" "name": "podinfo", "name": "PODNAME", "name": "postgres" "name": "primary-volume" "name": "primary-volume" "name": "primary-volume", "name": "primary-volume", "name": "REMOVE_BACKUP", "name": "REMOVE_DATA", "name": "REPLICA_NAME", "name": "report", "name": "RESTORE_OPTS", "name": "rmdata", "name": "root-volume" "name": "root-volume" "name": "root-volume", "name": "root-volume", "name": "some-name-11182", "name": "some-name-11182", "name": "some-name-11182-backrest-repo-config", "name": "some-name-11182-backrest-shared-repo", "name": "some-name-11182-backrest-shared-repo" "name": "some-name-11182-backrest-shared-repo", "name": "some-name-11182-backrest-shared-repo", "name": "some-name-11182-backrest-shared-repo", "name": "some-name-11182-bootstrap", "name": "some-name-11182-bootstrap", "name": "some-name-11182-custom-config" "name": "some-name-11182-custom-config" "name": "some-name-11182-full-sch-backup", "name": "some-name-11182-pgbouncer", "name": "some-name-11182-pgbouncer", "name": "some-name-11182-pgbouncer", "name": "some-name-11182-pgbouncer", "name": "some-name-11182-pgbouncer-cm" "name": "some-name-11182-pgbouncer-secret", "name": "some-name-11182-pgbouncer-secret", "name": "some-name-11182-pgbr-repo", "name": "some-name-11182","pg-cluster": "some-name-11182","deployment-name": "some-name-11182","pgouser": "admin","crunchy-pgha-scope": "some-name-11182","pgo-version": "1.7.0" "name": "some-name-11182-pgha-config", "name": "some-name-11182-pgha-config", "name": "some-name-11182-pgha-config", "name": "some-name-11182-repl1", "name": "some-name-11182-repl1", "name": "some-name-11182-repl2", "name": "some-name-11182-repl2", "name": "some-name-11182-replica","pg-cluster": "some-name-11182","deployment-name": "some-name-11182-repl2","pgo-version": "1.7.0" "name": "some-name-11182-rmdata-jurn", "name": "some-name-11182-rmdata-jurn", "name": "some-name-11182-stanza-create", "name": "some-name-1419", "name": "some-name-1419", "name": "some-name-1419-backrest-repo-config", "name": "some-name-1419-backrest-shared-repo", "name": "some-name-1419-backrest-shared-repo" "name": "some-name-1419-backrest-shared-repo", "name": "some-name-1419-backrest-shared-repo", "name": "some-name-1419-backrest-shared-repo", "name": "some-name-1419-bootstrap", "name": "some-name-1419-bootstrap", "name": "some-name-1419-custom-config" "name": "some-name-1419-custom-config" "name": "some-name-1419-full-sch-backup", "name": "some-name-1419-pgbouncer", "name": "some-name-1419-pgbouncer", "name": "some-name-1419-pgbouncer", "name": "some-name-1419-pgbouncer", "name": "some-name-1419-pgbouncer-cm" "name": "some-name-1419-pgbouncer-secret", "name": "some-name-1419-pgbouncer-secret", "name": "some-name-1419-pgbr-repo", "name": "some-name-1419-pgha-config", "name": "some-name-1419-pgha-config", "name": "some-name-1419-pgha-config", "name": "some-name-1419-repl1", "name": "some-name-1419-repl1", "name": "some-name-1419-repl2", "name": "some-name-1419-repl2", "name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl1","pgo-version": "1.7.0" "name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl1","pgo-version": "1.7.0" "name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl2","pgo-version": "1.7.0" "name": "some-name-1419-stanza-create", "name": "some-name-16172", "name": "some-name-16172", "name": "some-name-16172-backrest-repo-config", "name": "some-name-16172-backrest-shared-repo", "name": "some-name-16172-backrest-shared-repo" "name": "some-name-16172-backrest-shared-repo", "name": "some-name-16172-backrest-shared-repo", "name": "some-name-16172-backrest-shared-repo", "name": "some-name-16172-bootstrap", "name": "some-name-16172-bootstrap", "name": "some-name-16172-custom-config" "name": "some-name-16172-custom-config" "name": "some-name-16172-incr-sch-backup", "name": "some-name-16172-pgbouncer", "name": "some-name-16172-pgbouncer", "name": "some-name-16172-pgbouncer", "name": "some-name-16172-pgbouncer", "name": "some-name-16172-pgbouncer-cm" "name": "some-name-16172-pgbouncer-secret", "name": "some-name-16172-pgbouncer-secret", "name": "some-name-16172-pgbr-repo", "name": "some-name-16172-pgha-config", "name": "some-name-16172-pgha-config", "name": "some-name-16172-pgha-config", "name": "some-name-16172-repl1", "name": "some-name-16172-repl1", "name": "some-name-16172-repl2", "name": "some-name-16172-repl2", "name": "some-name-16172-replica","pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl1","pgo-version": "1.7.0" "name": "some-name-16172-replica","pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl2","pgo-version": "1.7.0" "name": "some-name-16172-rmdata-quxb", "name": "some-name-16172-rmdata-quxb", "name": "some-name-16172-stanza-create", "name": "some-name-17978", "name": "some-name-17978", "name": "some-name-17978-backrest-repo-config", "name": "some-name-17978-backrest-shared-repo", "name": "some-name-17978-backrest-shared-repo" "name": "some-name-17978-backrest-shared-repo", "name": "some-name-17978-backrest-shared-repo", "name": "some-name-17978-backrest-shared-repo", "name": "some-name-17978-bootstrap", "name": "some-name-17978-bootstrap", "name": "some-name-17978-custom-config" "name": "some-name-17978-custom-config" "name": "some-name-17978-diff-sch-backup", "name": "some-name-17978-pgbouncer", "name": "some-name-17978-pgbouncer", "name": "some-name-17978-pgbouncer", "name": "some-name-17978-pgbouncer", "name": "some-name-17978-pgbouncer-cm" "name": "some-name-17978-pgbouncer-secret", "name": "some-name-17978-pgbouncer-secret", "name": "some-name-17978-pgbr-repo", "name": "some-name-17978-pgha-config", "name": "some-name-17978-pgha-config", "name": "some-name-17978-pgha-config", "name": "some-name-17978-repl1", "name": "some-name-17978-repl1", "name": "some-name-17978-repl2", "name": "some-name-17978-repl2", "name": "some-name-17978-rmdata-kkbf", "name": "some-name-17978-rmdata-kkbf", "name": "some-name-17978-stanza-create", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "ssh-config", "name": "sshd", "name": "sshd", "name": "sshd", "name": "sshd", "name": "tls-pgbouncer" "name": "tls-pgbouncer", "name": "tls-replication" "name": "tls-replication", "name": "tls-server" "name": "tls-server" "name": "tls-server", "name": "tls-server", "name": "tmp" "name": "tmp" "name": "tmp" "name": "tmp", "name": "tmp", "name": "tmp", "name": "user-volume" "name": "user-volume" "name": "user-volume", "name": "user-volume", "nodePort": 0 "operator": "Exists" "operator": "In", "/opt/crunchy/bin/postgres-ha/health/pgha-liveness.sh" "/opt/crunchy/bin/postgres-ha/health/pgha-readiness.sh" "optional": true "optional": true "path": "annotations" "path": "cpu_limit", "path": "cpu_request", "path": "labels" "path": "mem_limit", "path": "mem_request", "path": "ssh_config" "path": "ssh_config" "path": "ssh_config" "path": "tls-replication.crt" "path": "tls-replication.key" "periodSeconds": 15, "persistentVolumeClaim": { "persistentVolumeClaim":{"claimName":"some-name-11182"} "persistentVolumeClaim":{"claimName":"some-name-11182"} "persistentVolumeClaim":{"claimName":"some-name-11182-repl1"} "persistentVolumeClaim":{"claimName":"some-name-11182-repl2"} "persistentVolumeClaim":{"claimName":"some-name-1419"} "persistentVolumeClaim":{"claimName":"some-name-1419"} "persistentVolumeClaim":{"claimName":"some-name-1419-repl1"} "persistentVolumeClaim":{"claimName":"some-name-1419-repl2"} "persistentVolumeClaim":{"claimName":"some-name-16172"} "persistentVolumeClaim":{"claimName":"some-name-16172"} "persistentVolumeClaim":{"claimName":"some-name-16172-repl1"} "persistentVolumeClaim":{"claimName":"some-name-16172-repl2"} "persistentVolumeClaim":{"claimName":"some-name-17978"} "persistentVolumeClaim":{"claimName":"some-name-17978"} "persistentVolumeClaim":{"claimName":"some-name-17978-repl1"} "persistentVolumeClaim":{"claimName":"some-name-17978-repl2"} "pg-cluster": "some-name-11182" "pg-cluster": "some-name-11182", "pg-cluster": "some-name-11182" "pg-cluster": "some-name-11182", "pg-cluster": "some-name-11182" "pg-cluster": "some-name-11182", "pg-cluster": "some-name-11182","deployment-name": "some-name-11182-repl1","pgo-version": "1.7.0","name": "some-name-11182-replica" "pg-cluster": "some-name-1419" "pg-cluster": "some-name-1419", "pg-cluster": "some-name-1419" "pg-cluster": "some-name-1419", "pg-cluster": "some-name-1419" "pg-cluster": "some-name-1419", "pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl2","pgo-version": "1.7.0","name": "some-name-1419-replica" "pg-cluster": "some-name-16172" "pg-cluster": "some-name-16172", "pg-cluster": "some-name-16172" "pg-cluster": "some-name-16172", "pg-cluster": "some-name-16172" "pg-cluster": "some-name-16172", "pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl1","pgo-version": "1.7.0","name": "some-name-16172-replica" "pg-cluster": "some-name-17978" "pg-cluster": "some-name-17978", "pg-cluster": "some-name-17978" "pg-cluster": "some-name-17978", "pg-cluster": "some-name-17978" "pg-cluster": "some-name-17978", "pg-cluster": "some-name-17978","deployment-name": "some-name-17978-repl2","pgo-version": "1.7.0","name": "some-name-17978-replica" "pgha-bootstrap": "some-name-11182", "pgha-bootstrap": "some-name-11182", "pgha-bootstrap": "some-name-1419", "pgha-bootstrap": "some-name-1419", "pgha-bootstrap": "some-name-16172", "pgha-bootstrap": "some-name-16172", "pgha-bootstrap": "some-name-17978", "pgha-bootstrap": "some-name-17978", "pgo-backrest-job": "true", "pgo-backrest-job": "true", "pgo-backrest-repo": "true" "pgo-backrest-repo": "true", "pgo-backrest-repo": "true" "pgo-backrest": "true", "pgo-pg-database": "true", "pgo-pg-database": "true", "pgouser": "admin","crunchy-pgha-scope": "some-name-1419","pgo-version": "1.7.0","name": "some-name-1419","pg-cluster": "some-name-1419","deployment-name": "some-name-1419" "pgouser": "admin","crunchy-pgha-scope": "some-name-17978","pgo-version": "1.7.0","name": "some-name-17978","pg-cluster": "some-name-17978","deployment-name": "some-name-17978" "pgouser": "admin","crunchy-pgha-scope": "some-name-17978","pgo-version": "1.7.0","name": "some-name-17978","pg-cluster": "some-name-17978","deployment-name": "some-name-17978" "pgo-version": "1.7.0","name": "some-name-11182","pg-cluster": "some-name-11182","deployment-name": "some-name-11182","pgouser": "admin","crunchy-pgha-scope": "some-name-11182" "pgo-version": "1.7.0","name": "some-name-11182-replica","pg-cluster": "some-name-11182","deployment-name": "some-name-11182-repl1" "pgo-version": "1.7.0","name": "some-name-11182-replica","pg-cluster": "some-name-11182","deployment-name": "some-name-11182-repl2" "pgo-version": "1.7.0","name": "some-name-11182-replica","pg-cluster": "some-name-11182","deployment-name": "some-name-11182-repl2" "pgo-version": "1.7.0","name": "some-name-1419","pg-cluster": "some-name-1419","deployment-name": "some-name-1419","pgouser": "admin","crunchy-pgha-scope": "some-name-1419" "pgo-version": "1.7.0","name": "some-name-1419","pg-cluster": "some-name-1419","deployment-name": "some-name-1419","pgouser": "admin","crunchy-pgha-scope": "some-name-1419" "pgo-version": "1.7.0","name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl1" "pgo-version": "1.7.0","name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl2" "pgo-version": "1.7.0","name": "some-name-1419-replica","pg-cluster": "some-name-1419","deployment-name": "some-name-1419-repl2" "pgo-version": "1.7.0","name": "some-name-16172","pg-cluster": "some-name-16172","deployment-name": "some-name-16172","pgouser": "admin","crunchy-pgha-scope": "some-name-16172" "pgo-version": "1.7.0","name": "some-name-16172","pg-cluster": "some-name-16172","deployment-name": "some-name-16172","pgouser": "admin","crunchy-pgha-scope": "some-name-16172" "pgo-version": "1.7.0","name": "some-name-16172-replica","pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl1" "pgo-version": "1.7.0","name": "some-name-16172-replica","pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl1" "pgo-version": "1.7.0","name": "some-name-16172-replica","pg-cluster": "some-name-16172","deployment-name": "some-name-16172-repl2" "pgo-version": "1.7.0","name": "some-name-17978","pg-cluster": "some-name-17978","deployment-name": "some-name-17978","pgouser": "admin","crunchy-pgha-scope": "some-name-17978" "pgo-version": "1.7.0","name": "some-name-17978","pg-cluster": "some-name-17978","deployment-name": "some-name-17978","pgouser": "admin","crunchy-pgha-scope": "some-name-17978" "pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978","deployment-name": "some-name-17978-repl1" "pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978","deployment-name": "some-name-17978-repl1" "pgo-version": "1.7.0","name": "some-name-17978-replica","pg-cluster": "some-name-17978","deployment-name": "some-name-17978-repl2" "pg-pod-anti-affinity": "preferred", "pg-pod-anti-affinity": "preferred", "pgrmdata": "true", "pgrmdata": "true", "podAffinityTerm": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "podAntiAffinity": { "port": 2022, "ports": [{ "ports": [{ "preferredDuringSchedulingIgnoredDuringExecution": [ "privileged": false, "projected": { "projected": { "projected": { "projected": { "projected": { "sources": [] } "projected": { "sources": [] } "protocol": "TCP", "protocol": "TCP", "readinessProbe": { "readOnly": false "readOnly": false "readOnly": false "readOnlyRootFilesystem": true "readOnly": true "readOnly": true "readOnly": true "readOnly": true "ReadWriteOnce" "replicas": 0, "replicas": 1, "replicas": 3, "requests": { "requests": { "resourceFieldRef": { "resource": "limits.cpu" "resource": "limits.memory" "resource": "requests.cpu" "resource": "requests.memory" "resources": { "resources": { "restartPolicy": "Always", "restartPolicy": "Never" "rollingUpdate": { "runAsNonRoot": true "secret": { "secret": { "secret": { "secret": { "secret": { "secretKeyRef": { "secretKeyRef": { "secretName": "some-name-11182-backrest-repo-config" "secretName": "some-name-11182-backrest-repo-config", "secretName": "some-name-11182-backrest-repo-config" "secretName": "some-name-11182-backrest-repo-config", "secretName": "some-name-11182-bootstrap-backrest-repo-config" "secretName": "some-name-11182-bootstrap-backrest-repo-config", "secretName": "some-name-11182-postgres-secret" "secretName": "some-name-11182-postgres-secret" "secretName": "some-name-11182-primaryuser-secret" "secretName": "some-name-11182-primaryuser-secret" "secretName": "some-name-11182-some-name-secret" "secretName": "some-name-11182-some-name-secret" "secretName": "some-name-1419-backrest-repo-config" "secretName": "some-name-1419-backrest-repo-config", "secretName": "some-name-1419-backrest-repo-config" "secretName": "some-name-1419-backrest-repo-config", "secretName": "some-name-1419-bootstrap-backrest-repo-config" "secretName": "some-name-1419-bootstrap-backrest-repo-config", "secretName": "some-name-1419-postgres-secret" "secretName": "some-name-1419-postgres-secret" "secretName": "some-name-1419-primaryuser-secret" "secretName": "some-name-1419-primaryuser-secret" "secretName": "some-name-1419-some-name-secret" "secretName": "some-name-1419-some-name-secret" "secretName": "some-name-16172-backrest-repo-config" "secretName": "some-name-16172-backrest-repo-config", "secretName": "some-name-16172-backrest-repo-config" "secretName": "some-name-16172-backrest-repo-config", "secretName": "some-name-16172-bootstrap-backrest-repo-config" "secretName": "some-name-16172-bootstrap-backrest-repo-config", "secretName": "some-name-16172-postgres-secret" "secretName": "some-name-16172-postgres-secret" "secretName": "some-name-16172-primaryuser-secret" "secretName": "some-name-16172-primaryuser-secret" "secretName": "some-name-16172-some-name-secret" "secretName": "some-name-16172-some-name-secret" "secretName": "some-name-17978-backrest-repo-config" "secretName": "some-name-17978-backrest-repo-config", "secretName": "some-name-17978-backrest-repo-config" "secretName": "some-name-17978-backrest-repo-config", "secretName": "some-name-17978-bootstrap-backrest-repo-config" "secretName": "some-name-17978-bootstrap-backrest-repo-config", "secretName": "some-name-17978-postgres-secret" "secretName": "some-name-17978-postgres-secret" "secretName": "some-name-17978-primaryuser-secret" "secretName": "some-name-17978-primaryuser-secret" "secretName": "some-name-17978-some-name-secret" "secretName": "some-name-17978-some-name-secret" "securityContext": { "securityContext": { "securityContext": {}, "securityContext": {"fsGroup": 26,"supplementalGroups": [1001]}, "securityContext": {"runAsNonRoot":true,"fsGroup":26}, "selector": { "selector": { "serviceAccountName": "pgo-backrest", "serviceAccountName": "pgo-default", "serviceAccountName": "pgo-pg", "serviceAccountName": "pgo-target", "service-name": "some-name-11182-backrest-shared-repo", "service-name": "some-name-11182-backrest-shared-repo", "service-name": "some-name-11182-pgbouncer", "service-name": "some-name-11182-pgbouncer", "service-name": "some-name-11182-pgbouncer", "service-name": "some-name-1419-backrest-shared-repo", "service-name": "some-name-1419-backrest-shared-repo", "service-name": "some-name-1419-pgbouncer", "service-name": "some-name-1419-pgbouncer", "service-name": "some-name-1419-pgbouncer", "service-name": "some-name-16172-backrest-shared-repo", "service-name": "some-name-16172-backrest-shared-repo", "service-name": "some-name-16172-pgbouncer", "service-name": "some-name-16172-pgbouncer", "service-name": "some-name-16172-pgbouncer", "service-name": "some-name-17978-backrest-shared-repo", "service-name": "some-name-17978-backrest-shared-repo", "service-name": "some-name-17978-pgbouncer", "service-name": "some-name-17978-pgbouncer", "service-name": "some-name-17978-pgbouncer", "sessionAffinity": "None" "sizeLimit": "128Ki" "sizeLimit": "16Mi" "sizeLimit": "16Mi" "sizeLimit": "1Mi" "sizeLimit": "2Mi" "sizeLimit": "64Mi" "sizeLimit": "64Mi" "some-name-11182" "some-name-1419" "some-name-16172" "some-name-17978" "sources": [ "sources": [ "sources": [ "sources": [ "spec": { "spec": { "spec": { "storage": "1Gi" "strategy": { "targetPort": 2022, "template": { time="2024-05-24T17:39:09Z" level=warning msg="PGAdminStorage setting not set, will use PrimaryStorage setting" func="github.com/percona/percona-postgresql-operator/internal/config.(*PgoConfig).Validate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/config/pgoconfig.go:337" version=1.7.0 time="2024-05-24T17:39:56Z" level=error msg="get deployment some-name-11182-repl1: deployments.apps \"some-name-11182-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T17:40:01Z" level=error msg="get deployment some-name-11182-repl2: deployments.apps \"some-name-11182-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T17:41:10Z" level=error msg="Error completing workflow some-name-11182-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T17:41:10Z" level=error msg="pgtasks.pg.percona.com \"some-name-11182-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T17:43:31Z" level=warning msg="crv1 pgreplica some-name-11182-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T17:43:32Z" level=warning msg="crv1 pgreplica some-name-11182-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T17:47:45Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.7.0 time="2024-05-24T17:47:45Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.7.0 time="2024-05-24T17:48:33Z" level=error msg="configmaps \"some-name-11182-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.7.0 time="2024-05-24T17:48:33Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-11182-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.7.0 time="2024-05-24T17:49:08Z" level=error msg="Error completing workflow some-name-11182-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T17:49:08Z" level=error msg="pgtasks.pg.percona.com \"some-name-11182-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T17:49:09Z" level=warning msg="deployments.apps \"some-name-11182-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.7.0 time="2024-05-24T17:51:33Z" level=warning msg="crv1 pgreplica some-name-11182-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T17:51:33Z" level=warning msg="crv1 pgreplica some-name-11182-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T17:53:55Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:53:58Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:53:59Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:00Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:02Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:04Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:07Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:11Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:15Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:17Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:18Z" level=error msg="pgclusters.pg.percona.com \"some-name-11182\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T17:54:23Z" level=error msg="jobs.batch \"some-name-11182-rmdata-jurn\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.7.0 time="2024-05-24T17:54:47Z" level=error msg="get deployment some-name-17978-repl1: deployments.apps \"some-name-17978-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T17:54:52Z" level=error msg="get deployment some-name-17978-repl2: deployments.apps \"some-name-17978-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T17:55:37Z" level=error msg="Error completing workflow some-name-17978-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T17:55:37Z" level=error msg="pgtasks.pg.percona.com \"some-name-17978-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T17:57:40Z" level=warning msg="crv1 pgreplica some-name-17978-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T17:57:40Z" level=warning msg="crv1 pgreplica some-name-17978-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:01:29Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.7.0 time="2024-05-24T18:01:29Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.7.0 time="2024-05-24T18:02:04Z" level=error msg="configmaps \"some-name-17978-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.7.0 time="2024-05-24T18:02:04Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-17978-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.7.0 time="2024-05-24T18:02:45Z" level=error msg="Error completing workflow some-name-17978-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T18:02:45Z" level=error msg="pgtasks.pg.percona.com \"some-name-17978-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T18:02:46Z" level=warning msg="deployments.apps \"some-name-17978-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.7.0 time="2024-05-24T18:04:59Z" level=warning msg="crv1 pgreplica some-name-17978-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:04:59Z" level=warning msg="crv1 pgreplica some-name-17978-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:05:10Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:11Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-17978-pgha-config (namespace scheduled-backup-13036),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.7.0 time="2024-05-24T18:05:11Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:12Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:13Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:14Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:16Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:18Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:19Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:45Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:47Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:57Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:05:58Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:06:02Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:06:04Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:06:05Z" level=error msg="pgclusters.pg.percona.com \"some-name-17978\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:06:10Z" level=error msg="jobs.batch \"some-name-17978-rmdata-kkbf\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.7.0 time="2024-05-24T18:06:38Z" level=error msg="get deployment some-name-16172-repl1: deployments.apps \"some-name-16172-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T18:06:43Z" level=error msg="get deployment some-name-16172-repl2: deployments.apps \"some-name-16172-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T18:07:23Z" level=error msg="Error completing workflow some-name-16172-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T18:07:23Z" level=error msg="pgtasks.pg.percona.com \"some-name-16172-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T18:09:30Z" level=warning msg="crv1 pgreplica some-name-16172-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:09:30Z" level=warning msg="crv1 pgreplica some-name-16172-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:16:03Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.7.0 time="2024-05-24T18:16:03Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.7.0 time="2024-05-24T18:16:42Z" level=error msg="configmaps \"some-name-16172-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.7.0 time="2024-05-24T18:16:42Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-16172-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.7.0 time="2024-05-24T18:17:23Z" level=error msg="Error completing workflow some-name-16172-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T18:17:23Z" level=error msg="pgtasks.pg.percona.com \"some-name-16172-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T18:17:24Z" level=warning msg="deployments.apps \"some-name-16172-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.7.0 time="2024-05-24T18:19:40Z" level=warning msg="crv1 pgreplica some-name-16172-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:19:40Z" level=warning msg="crv1 pgreplica some-name-16172-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:19:50Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:19:53Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:19:54Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:19:55Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:19:56Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:19:57Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:11Z" level=error msg="ConfigMap Controller: cannot find pgcluster for configMap some-name-16172-pgha-config (namespace scheduled-backup-13036),ignoring" func="github.com/percona/percona-postgresql-operator/internal/controller/configmap.(*Controller).handleConfigMapSync()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/configmap/synchandler.go:55" version=1.7.0 time="2024-05-24T18:20:26Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:28Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:31Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:35Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:36Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:39Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:41Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:42Z" level=error msg="pgclusters.pg.percona.com \"some-name-16172\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).onUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/podcontroller.go:103" version=1.7.0 time="2024-05-24T18:20:47Z" level=error msg="jobs.batch \"some-name-16172-rmdata-quxb\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleRMDataUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/rmdatahandler.go:61" version=1.7.0 time="2024-05-24T18:21:11Z" level=error msg="get deployment some-name-1419-repl1: deployments.apps \"some-name-1419-repl1\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T18:21:16Z" level=error msg="get deployment some-name-1419-repl2: deployments.apps \"some-name-1419-repl2\" not found" func="github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica.CreateReplicaResource()" file="/go/src/github.com/percona/percona-postgresql-operator/percona/controllers/pgreplica/pgreplica.go:40" version=1.7.0 time="2024-05-24T18:21:56Z" level=error msg="Error completing workflow some-name-1419-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T18:21:56Z" level=error msg="pgtasks.pg.percona.com \"some-name-1419-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T18:24:05Z" level=warning msg="crv1 pgreplica some-name-1419-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:24:05Z" level=warning msg="crv1 pgreplica some-name-1419-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:24:11Z" level=error func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).GetLocalConfigFromCluster()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:332" version=1.7.0 time="2024-05-24T18:24:11Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/kubeapi.ExecToPodThroughAPI()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/kubeapi/exec.go:76" version=1.7.0 time="2024-05-24T18:24:11Z" level=error msg="unable to upgrade connection: container not found (\"database\")" func="github.com/percona/percona-postgresql-operator/internal/operator/config.(*LocalDB).Sync.func2()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/config/localdb.go:173" version=1.7.0 time="2024-05-24T18:27:56Z" level=error msg="pgtask Controller: restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/controller/pgtask.(*Controller).handleBackrestRestore()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pgtask/backresthandler.go:72" version=1.7.0 time="2024-05-24T18:27:56Z" level=error msg="restore workflow error: workflow not found" func="github.com/percona/percona-postgresql-operator/internal/operator/backrest.UpdateWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/backrest/restore.go:301" version=1.7.0 time="2024-05-24T18:28:37Z" level=error msg="configmaps \"some-name-1419-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/util.ToggleAutoFailover()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/util/failover.go:273" version=1.7.0 time="2024-05-24T18:28:37Z" level=warning msg="jobController unable to toggle autofail during bootstrap, cluster could initialize in a paused state: configmaps \"some-name-1419-config\" not found" func="github.com/percona/percona-postgresql-operator/internal/controller/job.(*Controller).handleBootstrapUpdate()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/job/bootstraphandler.go:80" version=1.7.0 time="2024-05-24T18:29:41Z" level=error msg="Error completing workflow some-name-1419-createcluster" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:48" version=1.7.0 time="2024-05-24T18:29:41Z" level=error msg="pgtasks.pg.percona.com \"some-name-1419-createcluster\" not found" func="github.com/percona/percona-postgresql-operator/internal/operator/task.completeWorkflow()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/task/workflow.go:49" version=1.7.0 time="2024-05-24T18:29:42Z" level=warning msg="deployments.apps \"some-name-1419-pgbouncer\" already exists" func="github.com/percona/percona-postgresql-operator/internal/controller/pod.(*Controller).handleBootstrapInit()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/controller/pod/inithandler.go:187" version=1.7.0 time="2024-05-24T18:31:57Z" level=warning msg="crv1 pgreplica some-name-1419-repl1 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 time="2024-05-24T18:31:57Z" level=warning msg="crv1 pgreplica some-name-1419-repl2 is already marked complete, will not recreate" func="github.com/percona/percona-postgresql-operator/internal/operator/cluster.ScaleBase()" file="/go/src/github.com/percona/percona-postgresql-operator/internal/operator/cluster/cluster.go:535" version=1.7.0 "timeoutSeconds": 10 "topologyKey": "kubernetes.io/hostname" "type": "ClusterIP", "type": "RollingUpdate", "value": "" "value": "5432" "value": "5432" "value": "/backrestrepo/some-name-11182-backrest-shared-repo" "value": "/backrestrepo/some-name-11182-backrest-shared-repo" "value": "/backrestrepo/some-name-1419-backrest-shared-repo" "value": "/backrestrepo/some-name-1419-backrest-shared-repo" "value": "/backrestrepo/some-name-16172-backrest-shared-repo" "value": "/backrestrepo/some-name-16172-backrest-shared-repo" "value": "/backrestrepo/some-name-17978-backrest-shared-repo" "value": "/backrestrepo/some-name-17978-backrest-shared-repo" "value": "backup" "value": "crunchy-pgha-scope" "value": "db" "value": "db" "value": " --db-host=10.174.224.26 --db-path=/pgdata/some-name-1419" "value": " --db-host=10.174.225.15 --db-path=/pgdata/some-name-11182" "value": " --db-host=10.174.225.21 --db-path=/pgdata/some-name-17978" "value": " --db-host=10.174.226.13 --db-path=/pgdata/some-name-17978" "value": " --db-host=10.174.226.21 --db-path=/pgdata/some-name-16172" "value": " --db-host=10.174.226.25 --db-path=/pgdata/some-name-16172" "value": " --db-host=10.174.226.26 --db-path=/pgdata/some-name-1419" "value": " --db-host=10.174.226.7 --db-path=/pgdata/some-name-11182" "value": "false" "value": "false" "valueFrom": { "valueFrom": { "value": "INFO" "value": "path" "value": "pgbackrest" "value": "pgbackrest_init" "value": "pgbackrest-repo" "value": "/pgdata/some-name-11182" "value": "/pgdata/some-name-11182" "value": "/pgdata/some-name-11182-repl1" "value": "/pgdata/some-name-11182-repl1" "value": "/pgdata/some-name-11182-repl2" "value": "/pgdata/some-name-11182-repl2" "value": "/pgdata/some-name-1419" "value": "/pgdata/some-name-1419" "value": "/pgdata/some-name-1419-repl1" "value": "/pgdata/some-name-1419-repl1" "value": "/pgdata/some-name-1419-repl2" "value": "/pgdata/some-name-1419-repl2" "value": "/pgdata/some-name-16172" "value": "/pgdata/some-name-16172" "value": "/pgdata/some-name-16172-repl1" "value": "/pgdata/some-name-16172-repl1" "value": "/pgdata/some-name-16172-repl2" "value": "/pgdata/some-name-16172-repl2" "value": "/pgdata/some-name-17978" "value": "/pgdata/some-name-17978" "value": "/pgdata/some-name-17978-repl1" "value": "/pgdata/some-name-17978-repl1" "value": "/pgdata/some-name-17978-repl2" "value": "/pgdata/some-name-17978-repl2" "value": "pg-operator-testing" "value": "posix" "value": "posix" "value": "postgres" "values": [ "value": "s3.amazonaws.com" "value": "some-name-11182" "value": "some-name-11182-backrest-shared-repo" "value": "some-name-11182-backrest-shared-repo-5d9f755494-ghjcv" "value": "some-name-11182-backrest-shared-repo-5d9f755494-nkrzs" "value": "some-name-11182-backrest-shared-repo.scheduled-backup-13036.svc.cluster.local." "value": "some-name-1419" "value": "some-name-1419-backrest-shared-repo" "value": "some-name-1419-backrest-shared-repo-bcf445bb6-crqp4" "value": "some-name-1419-backrest-shared-repo-bcf445bb6-xsq2c" "value": "some-name-1419-backrest-shared-repo.scheduled-backup-13036.svc.cluster.local." "value": "some-name-16172" "value": "some-name-16172-backrest-shared-repo" "value": "some-name-16172-backrest-shared-repo-5f7d45f6f4-lvw62" "value": "some-name-16172-backrest-shared-repo-5f7d45f6f4-ptfkl" "value": "some-name-16172-backrest-shared-repo.scheduled-backup-13036.svc.cluster.local." "value": "some-name-17978" "value": "some-name-17978-backrest-shared-repo" "value": "some-name-17978-backrest-shared-repo-f9848f96b-l4fvt" "value": "some-name-17978-backrest-shared-repo-f9848f96b-x82q9" "value": "some-name-17978-backrest-shared-repo.scheduled-backup-13036.svc.cluster.local." "value": "/sshd/aws-s3-ca.crt" "value": "stanza-create" "value": "--stanza=db --type=diff --repo1-retention-diff=3 --db-host=10.174.225.21 --db-path=/pgdata/some-name-17978" "value": "--stanza=db --type=diff --repo1-retention-diff=3 --db-host=10.174.226.13 --db-path=/pgdata/some-name-17978" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.174.224.26 --db-path=/pgdata/some-name-1419" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.174.225.15 --db-path=/pgdata/some-name-11182" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.174.226.26 --db-path=/pgdata/some-name-1419" "value": "--stanza=db --type=full --repo1-retention-full=3 --db-host=10.174.226.7 --db-path=/pgdata/some-name-11182" "value": "--stanza=db --type=incr --db-host=10.174.226.21 --db-path=/pgdata/some-name-16172" "value": "/tmp" "value": "/tmp" "value": "/tmp/nss_wrapper/pgbackrest-repo/group" "value": "/tmp/nss_wrapper/pgbackrest-repo/passwd" "value": "/tmp/nss_wrapper/postgres/group" "value": "/tmp/nss_wrapper/postgres/passwd" "value": "true" "value": "true" "value": "--type=full --db-host=10.174.224.26 --db-path=/pgdata/some-name-1419" "value": "--type=full --db-host=10.174.225.15 --db-path=/pgdata/some-name-11182" "value": "--type=full --db-host=10.174.225.21 --db-path=/pgdata/some-name-17978" "value": "--type=full --db-host=10.174.226.13 --db-path=/pgdata/some-name-17978" "value": "--type=full --db-host=10.174.226.21 --db-path=/pgdata/some-name-16172" "value": "--type=full --db-host=10.174.226.25 --db-path=/pgdata/some-name-16172" "value": "--type=full --db-host=10.174.226.26 --db-path=/pgdata/some-name-1419" "value": "--type=full --db-host=10.174.226.7 --db-path=/pgdata/some-name-11182" "value": "--type=xid --target=\"773\" --repo-type=s3 --no-repo1-s3-verify-tls --target-action=promote" "value": "us-east-1" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/lib64/libnss_wrapper.so" "value": "/usr/local/bin/archive-push-s3.sh" "value": "{vendor: \"crunchydata\"}" "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "vendor": "crunchydata" "vendor": "crunchydata", "volumeMounts": [ "volumeMounts": [{ "volumes": [ "volumes": [{ "weight": 1, + destroy_operator + kubectl_bin delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.2h3NoNlACv ++ mktemp + local LAST_ERR=/tmp/tmp.viJY0Xtb3b + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete -f /tmp/tmp.7cfNRxU4oI/operator.yaml + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.2h3NoNlACv + cat /tmp/tmp.viJY0Xtb3b Error from server (NotFound): error when deleting "/tmp/tmp.7cfNRxU4oI/operator.yaml": serviceaccounts "pgo-deployer-sa" not found Error from server (NotFound): error when deleting "/tmp/tmp.7cfNRxU4oI/operator.yaml": clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found Error from server (NotFound): error when deleting "/tmp/tmp.7cfNRxU4oI/operator.yaml": configmaps "pgo-deployer-cm" not found Error from server (NotFound): error when deleting "/tmp/tmp.7cfNRxU4oI/operator.yaml": clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-crb" not found Error from server (NotFound): error when deleting "/tmp/tmp.7cfNRxU4oI/operator.yaml": jobs.batch "pgo-deploy" not found + rm /tmp/tmp.2h3NoNlACv /tmp/tmp.viJY0Xtb3b + return 1 + true + yq w -d4 /tmp/tmp.7cfNRxU4oI/operator.yaml 'spec.template.spec.containers[0].env[0].value' uninstall + kubectl_bin apply -f /tmp/tmp.7cfNRxU4oI/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.9mvzWz2WKO ++ mktemp + local LAST_ERR=/tmp/tmp.TaUPowRTqQ + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl apply -f /tmp/tmp.7cfNRxU4oI/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.9mvzWz2WKO serviceaccount/pgo-deployer-sa created clusterrole.rbac.authorization.k8s.io/pgo-deployer-cr created configmap/pgo-deployer-cm created clusterrolebinding.rbac.authorization.k8s.io/pgo-deployer-crb created job.batch/pgo-deploy created + cat /tmp/tmp.TaUPowRTqQ + rm /tmp/tmp.9mvzWz2WKO /tmp/tmp.TaUPowRTqQ + return 0 + wait_job_completion pgo-deploy true + local job=pgo-deploy + local mandatory=true + local ns=scheduled-backup-13036 + retry=0 + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' pgo-deploy+ retry=0 ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 1 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 2 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 3 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 4 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 5 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 6 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 7 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 8 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 9 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 10 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 11 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 12 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 13 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 14 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 15 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 16 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 17 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 18 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 19 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 20 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 21 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 22 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 23 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 24 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 25 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 26 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 27 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 28 -ge 600 ']' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' ++ grep -io 'not found' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ '' == \1 ]] + sleep 1 + echo -n . .+ let retry+=1 + '[' 29 -ge 600 ']' ++ grep -io 'not found' ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.metadata.name}' + [[ '' == \n\o\t\ \f\o\u\n\d ]] ++ kubectl_bin -n scheduled-backup-13036 get job/pgo-deploy -o 'jsonpath={.status.succeeded}' + [[ 1 == \1 ]] + echo + kubectl_bin delete -f /tmp/tmp.7cfNRxU4oI/operator-destroy.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.heZhmBYfhu ++ mktemp + local LAST_ERR=/tmp/tmp.qSmba8ewGt + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete -f /tmp/tmp.7cfNRxU4oI/operator-destroy.yaml + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.heZhmBYfhu serviceaccount "pgo-deployer-sa" deleted clusterrole.rbac.authorization.k8s.io "pgo-deployer-cr" deleted configmap "pgo-deployer-cm" deleted clusterrolebinding.rbac.authorization.k8s.io "pgo-deployer-crb" deleted job.batch "pgo-deploy" deleted + cat /tmp/tmp.qSmba8ewGt + rm /tmp/tmp.heZhmBYfhu /tmp/tmp.qSmba8ewGt + return 0 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|openshift|^NAME' ++ mktemp + awk '{print$1}' + local LAST_OUT=/tmp/tmp.3r6G3oDPVK ++ mktemp + local LAST_ERR=/tmp/tmp.dUcIWOR6cP + local exit_status=0 + local timeout=4 + set +o errexit + xargs kubectl delete ns --grace-period=0 --force=true ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + set -o errexit + cat /tmp/tmp.3r6G3oDPVK + cat /tmp/tmp.dUcIWOR6cP + rm /tmp/tmp.3r6G3oDPVK /tmp/tmp.dUcIWOR6cP + return 0 Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "scheduled-backup-13036" force deleted + kubectl_bin delete clusterrolebindings pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.PGoSYtAXuK ++ mktemp + local LAST_ERR=/tmp/tmp.H2b9hICDnk + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterrolebindings pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.PGoSYtAXuK + cat /tmp/tmp.H2b9hICDnk Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.PGoSYtAXuK /tmp/tmp.H2b9hICDnk + return 1 + true + kubectl_bin delete clusterroles pgo-cluster-role pgo-deployer-cr ++ mktemp + local LAST_OUT=/tmp/tmp.vavbfAqtUI ++ mktemp + local LAST_ERR=/tmp/tmp.Un4OBSmqYM + local exit_status=0 + local timeout=4 + set +o errexit ++ seq 0 2 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 4 + for i in '$(seq 0 2)' + KUBECONFIG=/tmp/jen-pg-776-0087ef129-2-scheduled-backup + kubectl delete clusterroles pgo-cluster-role pgo-deployer-cr + exit_status=1 + [[ 1 != 0 ]] + sleep 8 + set -o errexit + cat /tmp/tmp.vavbfAqtUI + cat /tmp/tmp.Un4OBSmqYM Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-cluster-role" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "pgo-deployer-cr" not found + rm /tmp/tmp.vavbfAqtUI /tmp/tmp.Un4OBSmqYM + return 1 + true + rm -rf /tmp/tmp.7cfNRxU4oI